Make the loss / delay sim available as an output feature

Add binary api and debug cli support.

Rewrite for speed: enqueue vlib_buffer_t's to the wheel, instead of
memcpy'ing data. Quad-loop the output feature / x-connect (interior)
node. Prefetch wheel entries in the input node.

Save packet-generator-based unit-test setup in extras/nsim.

Simple config example:

set nsim delay 20 ms bandwidth 1 gbit packet-size 1024
nsim output-feature enable-disable GigabitEthernet3/0/0

Change-Id: I852a32d4eb596e7e2aa1d9b30bf3b53525e39fd1
Signed-off-by: Dave Barach <dave@barachs.net>c

extras/nsim/setup.nsim

@@ -0,0 +1,25 @@
+set term pag off
+
+loop cre
+
+set int ip address loop0 192.168.2.1/24
+set int state loop0 up
+set nsim delay 20 ms bandwidth 1 gbit packet-size 128
+comment { add drop-fraction 0.8 or some such}
+nsim output-feature enable-disable loop0
+
+packet-generator new {
+    name icmp
+    limit 0
+    size 128-128
+    interface local0
+    node ethernet-input
+    data {
+        IP4: 0001.dead.beef -> 0002.dead.beef
+        ICMP: 192.168.1.2 -> 192.168.2.2
+        incrementing 30
+    }
+}
+
+set int ip address pg0 192.168.1.1/24
+set ip arp loop0 192.168.2.2 0003.dead.beef

src/plugins/nsim/nsim.api

@@ -3,7 +3,7 @@
  * @brief VPP control-plane API messages for the network delay simulator
  */
 
-option version = "1.1.0";
+option version = "2.1.0";
 
 /** \brief enable / disable the network delay simulation cross-connect
     @param client_index - opaque cookie to identify the sender
@@ -12,7 +12,7 @@ option version = "1.1.0";
     @param sw_if_index0 - one interface to cross-connect
     @param sw_if_index1 - the other interface to cross-connect
 */
-autoreply define nsim_enable_disable 
+autoreply define nsim_cross_connect_enable_disable 
 {
   /* Client identifier, set from api_main.my_client_index */
   u32 client_index;
@@ -28,6 +28,27 @@ autoreply define nsim_enable_disable
   u32 sw_if_index1;
 };
 
+/** \brief enable / disable the network delay simulation output feature
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param enable_disable - enable or disable the feature
+    @param sw_if_index0 - interface 
+*/
+autoreply define nsim_output_feature_enable_disable 
+{
+  /* Client identifier, set from api_main.my_client_index */
+  u32 client_index;
+  
+  /* Arbitrary context, so client can match reply to request */
+  u32 context;
+  
+  /* Enable / disable the feature on the interfaces */
+  u8 enable_disable;
+  
+  /* Interface handles */
+  u32 sw_if_index;
+};
+
 /** \brief configure the network delay simulation cross-connect
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
@@ -49,3 +70,5 @@ autoreply define nsim_configure
   u64 bandwidth_in_bits_per_second;
   u32 packets_per_drop;
 };
+
+

src/plugins/nsim/nsim.h

@@ -25,15 +25,14 @@
 #include <vppinfra/hash.h>
 #include <vppinfra/error.h>
 
-#define WHEEL_ENTRY_DATA_SIZE 1536	/* an even multiple of 64, pls */
-
 typedef struct
 {
   f64 tx_time;
+  u32 rx_sw_if_index;
   u32 tx_sw_if_index;
-  u32 current_length;
-    CLIB_CACHE_LINE_ALIGN_MARK (pad);
-  u8 data[WHEEL_ENTRY_DATA_SIZE];
+  u32 output_next_index;
+  u32 buffer_index;
+  u32 pad;			/* pad to 32-bytes */
 } nsim_wheel_entry_t;
 
 typedef struct
@@ -54,12 +53,15 @@ typedef struct
   /* Two interfaces, cross-connected with delay */
   u32 sw_if_index0, sw_if_index1;
   u32 output_next_index0, output_next_index1;
+
+  /* N interfaces, using the output feature */
+  u32 *output_next_index_by_sw_if_index;
+
   /* Random seed for loss-rate simulation */
   u32 seed;
 
-  /* Per-thread buffer / scheduler wheels */
+  /* Per-thread scheduler wheels */
   nsim_wheel_t **wheel_by_thread;
-  u32 **buffer_indices_by_thread;
 
   /* Config parameters */
   f64 delay;

src/plugins/nsim/nsim_input.c

@@ -24,7 +24,7 @@
 typedef struct
 {
   f64 expired;
-  u32 tx_sw_if_index;
+  u32 next_index;
 } nsim_tx_trace_t;
 
 #ifndef CLIB_MARCH_VARIANT
@@ -36,22 +36,20 @@ format_nsim_tx_trace (u8 * s, va_list * args)
   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
   nsim_tx_trace_t *t = va_arg (*args, nsim_tx_trace_t *);
 
-  s = format (s, "NSIM: tx at %.6f sw_if_index %d",
-	      t->expired, t->tx_sw_if_index);
+  s = format (s, "NSIM: tx at %.6f next_index %d", t->expired, t->next_index);
   return s;
 }
 #endif /* CLIB_MARCH_VARIANT */
 
-#define foreach_nsim_tx_error                      \
-_(TX, "Packets transmitted")                    \
-_(DROPPED, "No buffer drops")
+#define foreach_nsim_tx_error                   \
+_(TRANSMITTED, "Packets transmitted")
 
 typedef enum
 {
 #define _(sym,str) NSIM_TX_ERROR_##sym,
   foreach_nsim_tx_error
 #undef _
-    NSIM_N_ERROR,
+    NSIM_TX_N_ERROR,
 } nsim_tx_error_t;
 
 #ifndef CLIB_MARCH_VARIANT
@@ -74,12 +72,9 @@ nsim_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 {
   nsim_main_t *nsm = &nsim_main;
   u32 my_thread_index = vm->thread_index;
-  u32 *my_buffer_cache = nsm->buffer_indices_by_thread[my_thread_index];
   nsim_wheel_t *wp = nsm->wheel_by_thread[my_thread_index];
-  u32 n_trace = vlib_get_trace_count (vm, node);
   f64 now = vlib_time_now (vm);
-  uword n_rx_packets = 0;
-  vlib_buffer_t *b0;
+  uword n_tx_packets = 0;
   u32 bi0, next0;
   u32 *to_next;
   u32 next_index;
@@ -93,85 +88,30 @@ nsim_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   /* First entry on the wheel isn't expired? */
   ep = wp->entries + wp->head;
   if (ep->tx_time > now)
-    return n_rx_packets;
+    return n_tx_packets;
 
-  /*
-   * We use per-thread buffer caches, so we need the freelist to
-   * initialize them...
-   */
   next_index = node->cached_next_index;
 
-  while (wp->cursize)
+  /* Be aware: this is not the usual coding pattern */
+  while (wp->cursize && ep->tx_time <= now)
     {
-      /* Be aware: this is not the usual coding pattern */
+      vlib_buffer_t *b0;
+
       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 
-      while (n_left_to_next > 0 && ep->tx_time <= now)
+      while (n_left_to_next > 0)
 	{
-	  /* Out of local buffer cache? */
-	  if (PREDICT_FALSE (_vec_len (my_buffer_cache) == 0))
-	    {
-	      u32 n =
-		vlib_buffer_alloc (vm, my_buffer_cache, VLIB_FRAME_SIZE);
-	      _vec_len (my_buffer_cache) = n;
-
-	      /* Ugh, drop the rest of the expired entries */
-	      if (n == 0)
-		{
-		  u32 drops = 0;
-		  while (ep->tx_time <= now && wp->cursize)
-		    {
-		      wp->head++;
-		      if (wp->head == wp->wheel_size)
-			wp->head = 0;
-		      ep = wp->entries + wp->head;
-		      wp->cursize--;
-		      drops++;
-		    }
-		  /* Count the drops */
-		  vlib_node_increment_counter (vm, node->node_index,
-					       NSIM_TX_ERROR_DROPPED, drops);
-		  /* Ship any pkts we already processed */
-		  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-		  return n_rx_packets + drops;
-		}
-	    }
-
-	  /* Allocate a buffer */
-	  bi0 = my_buffer_cache[_vec_len (my_buffer_cache) - 1];
-	  _vec_len (my_buffer_cache) -= 1;
+	  /* prefetch one line / 2 entries ahead */
+	  if ((((uword) ep) & (CLIB_CACHE_LINE_BYTES - 1)) == 0)
+	    CLIB_PREFETCH ((ep + 2), CLIB_CACHE_LINE_BYTES, LOAD);
+	  /* Pick up buffer from the wheel */
+	  bi0 = ep->buffer_index;
 
 	  to_next[0] = bi0;
 	  to_next += 1;
 	  n_left_to_next -= 1;
 
-	  b0 = vlib_get_buffer (vm, bi0);
-	  /* Initialize the buffer */
-
-	  b0->current_data = 0;
-	  b0->current_length = ep->current_length;
-
-	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
-
-	  if (PREDICT_FALSE (n_trace))
-	    {
-	      nsim_tx_trace_t *t0;
-	      vlib_trace_buffer (vm, node, next_index, b0,
-				 0 /* follow_chain */ );
-	      t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
-	      t0->expired = ep->tx_time;
-	      t0->tx_sw_if_index = ep->tx_sw_if_index;
-	    }
-
-	  /* Copy data from the ring */
-	  clib_memcpy_fast (b0->data, ep->data, ep->current_length);
-	  b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
-	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = ep->tx_sw_if_index;
-	  vnet_buffer (b0)->sw_if_index[VLIB_RX] =
-	    (ep->tx_sw_if_index == nsm->sw_if_index0) ? nsm->sw_if_index1 :
-	    nsm->sw_if_index0;
-	  next0 = (ep->tx_sw_if_index == nsm->sw_if_index0) ?
-	    nsm->output_next_index0 : nsm->output_next_index1;
+	  next0 = ep->output_next_index;
 
 	  /* verify speculative enqueue, maybe switch current next frame */
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
@@ -183,7 +123,19 @@ nsim_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	    wp->head = 0;
 	  wp->cursize--;
 	  ep = wp->entries + wp->head;
-	  n_rx_packets++;
+	  n_tx_packets++;
+
+	  if (is_trace)
+	    {
+	      b0 = vlib_get_buffer (vm, ep->buffer_index);
+	      if (b0->flags & VLIB_BUFFER_IS_TRACED)
+		{
+		  nsim_tx_trace_t *t =
+		    vlib_add_trace (vm, node, b0, sizeof (*t));
+		  t->expired = now;
+		  t->next_index = next0;
+		}
+	    }
 
 	  /* Out of ring entries? */
 	  if (PREDICT_FALSE (wp->cursize == 0))
@@ -196,7 +148,9 @@ nsim_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       if (ep->tx_time > now)
 	break;
     }
-  return n_rx_packets;
+  vlib_node_increment_counter (vm, node->node_index,
+			       NSIM_TX_ERROR_TRANSMITTED, n_tx_packets);
+  return n_tx_packets;
 }
 
 VLIB_NODE_FN (nsim_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
@@ -221,7 +175,7 @@ VLIB_REGISTER_NODE (nsim_input_node) =
 
   .format_trace = format_nsim_tx_trace,
 
-  .n_errors = NSIM_N_ERROR,
+  .n_errors = NSIM_TX_N_ERROR,
   .error_strings = nsim_tx_error_strings,
 };
 #endif /* CLIB_MARCH_VARIANT */

src/plugins/nsim/nsim_test.c

@@ -60,7 +60,8 @@ nsim_test_main_t nsim_test_main;
 #include <vlibapi/vat_helper_macros.h>
 
 #define foreach_standard_reply_retval_handler   \
-_(nsim_enable_disable_reply)                    \
+_(nsim_cross_connect_enable_disable_reply)      \
+_(nsim_output_feature_enable_disable_reply)     \
 _(nsim_configure_reply)
 
 #define _(n)                                            \
@@ -83,19 +84,22 @@ foreach_standard_reply_retval_handler;
  * Table of message reply handlers, must include boilerplate handlers
  * we just generated
  */
-#define foreach_vpe_api_reply_msg                       \
-_(NSIM_ENABLE_DISABLE_REPLY, nsim_enable_disable_reply) \
+#define foreach_vpe_api_reply_msg               \
+_(NSIM_CROSS_CONNECT_ENABLE_DISABLE_REPLY,      \
+nsim_cross_connect_enable_disable_reply)        \
+_(NSIM_OUTPUT_FEATURE_ENABLE_DISABLE_REPLY,     \
+nsim_output_feature_enable_disable_reply)       \
 _(NSIM_CONFIGURE_REPLY, nsim_configure_reply)
 
 static int
-api_nsim_enable_disable (vat_main_t * vam)
+api_nsim_cross_connect_enable_disable (vat_main_t * vam)
 {
   unformat_input_t *i = vam->input;
   int enable_disable = 1;
   u32 sw_if_index0 = ~0;
   u32 sw_if_index1 = ~0;
   u32 tmp;
-  vl_api_nsim_enable_disable_t *mp;
+  vl_api_nsim_cross_connect_enable_disable_t *mp;
   int ret;
 
   /* Parse args required to build the message */
@@ -128,7 +132,7 @@ api_nsim_enable_disable (vat_main_t * vam)
     }
 
   /* Construct the API message */
-  M (NSIM_ENABLE_DISABLE, mp);
+  M (NSIM_CROSS_CONNECT_ENABLE_DISABLE, mp);
   mp->sw_if_index0 = ntohl (sw_if_index0);
   mp->sw_if_index1 = ntohl (sw_if_index1);
   mp->enable_disable = enable_disable;
@@ -141,6 +145,47 @@ api_nsim_enable_disable (vat_main_t * vam)
   return ret;
 }
 
+static int
+api_nsim_output_feature_enable_disable (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  int enable_disable = 1;
+  u32 sw_if_index = ~0;
+  vl_api_nsim_output_feature_enable_disable_t *mp;
+  int ret;
+
+  /* Parse args required to build the message */
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index))
+	;
+      else if (unformat (i, "sw_if_index %d", &sw_if_index))
+	;
+      else if (unformat (i, "disable"))
+	enable_disable = 0;
+      else
+	break;
+    }
+
+  if (sw_if_index == ~0)
+    {
+      errmsg ("missing interface name / explicit sw_if_index number \n");
+      return -99;
+    }
+
+  /* Construct the API message */
+  M (NSIM_OUTPUT_FEATURE_ENABLE_DISABLE, mp);
+  mp->sw_if_index = ntohl (sw_if_index);
+  mp->enable_disable = enable_disable;
+
+  /* send it... */
+  S (mp);
+
+  /* Wait for a reply... */
+  W (ret);
+  return ret;
+}
+
 static uword
 unformat_delay (unformat_input_t * input, va_list * args)
 {
@@ -229,9 +274,10 @@ api_nsim_configure (vat_main_t * vam)
  * and that the data plane plugin processes
  */
 #define foreach_vpe_api_msg                                             \
-_(nsim_enable_disable,                                                  \
+_(nsim_cross_connect_enable_disable,                                    \
 "[<intfc0> | sw_if_index <swif0>] [<intfc1> | sw_if_index <swif1>] [disable]") \
-_(nsim_configure, "delay <time> bandwidth <bw> [packet-size <nn>]" \
+_(nsim_output_feature_enable_disable,"[<intfc> | sw_if_index <nnn> [disable]") \
+_(nsim_configure, "delay <time> bandwidth <bw> [packet-size <nn>]"      \
 "[packets-per-drop <nnnn>]")
 
 static void