nat: respect arc features (multi worker)

Type: fix
Ticket: VPP-1747

Change-Id: If282aae3e584d7017c200f897b99c8a37eb1b2e5
Signed-off-by: Filip Varga <fivarga@cisco.com>
(cherry picked from commit 9a6dc8a9376e7270331255861b3ead1045b40c6d)

src/plugins/nat/nat.c

@@ -39,8 +39,29 @@
 snat_main_t snat_main;
 
 /* *INDENT-OFF* */
-
 /* Hook up input features */
+VNET_FEATURE_INIT (nat_pre_in2out, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat-pre-in2out",
+  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
+};
+VNET_FEATURE_INIT (nat_pre_out2in, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat-pre-out2in",
+  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
+                               "ip4-dhcp-client-detect"),
+};
+VNET_FEATURE_INIT (snat_in2out_worker_handoff, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-in2out-worker-handoff",
+  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
+};
+VNET_FEATURE_INIT (snat_out2in_worker_handoff, static) = {
+  .arc_name = "ip4-unicast",
+  .node_name = "nat44-out2in-worker-handoff",
+  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
+                               "ip4-dhcp-client-detect"),
+};
 VNET_FEATURE_INIT (ip4_snat_in2out, static) = {
   .arc_name = "ip4-unicast",
   .node_name = "nat44-in2out",
@@ -89,17 +110,6 @@ VNET_FEATURE_INIT (ip4_nat44_ed_classify, static) = {
   .node_name = "nat44-ed-classify",
   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
 };
-VNET_FEATURE_INIT (ip4_snat_in2out_worker_handoff, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "nat44-in2out-worker-handoff",
-  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
-};
-VNET_FEATURE_INIT (ip4_snat_out2in_worker_handoff, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "nat44-out2in-worker-handoff",
-  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa",
-                               "ip4-dhcp-client-detect"),
-};
 VNET_FEATURE_INIT (ip4_nat_handoff_classify, static) = {
   .arc_name = "ip4-unicast",
   .node_name = "nat44-handoff-classify",
@@ -1775,18 +1785,20 @@ snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
       else if (sm->deterministic)
 	feature_name = is_inside ? "nat44-det-in2out" : "nat44-det-out2in";
       else if (sm->endpoint_dependent)
-	feature_name = is_inside ? "nat44-ed-in2out" : "nat44-ed-out2in";
+	{
+	  feature_name = is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
+	}
       else
 	feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
     }
 
   if (sm->fq_in2out_index == ~0 && !sm->deterministic && sm->num_workers > 1)
-    sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
+    sm->fq_in2out_index =
-						      NAT_FQ_NELTS);
+      vlib_frame_queue_main_init (sm->handoff_in2out_index, NAT_FQ_NELTS);
 
   if (sm->fq_out2in_index == ~0 && !sm->deterministic && sm->num_workers > 1)
-    sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
+    sm->fq_out2in_index =
-						      NAT_FQ_NELTS);
+      vlib_frame_queue_main_init (sm->handoff_out2in_index, NAT_FQ_NELTS);
 
   if (!is_inside)
     {
@@ -1844,8 +1856,8 @@ feature_set:
                 else if (sm->endpoint_dependent)
                   {
                     del_feature_name = "nat44-ed-classify";
-                    feature_name = !is_inside ?  "nat44-ed-in2out" :
+                    feature_name = !is_inside ?  "nat-pre-in2out" :
-                                                 "nat44-ed-out2in";
+                                                 "nat-pre-out2in";
                   }
                 else
                   {
@@ -1907,8 +1919,9 @@ feature_set:
               }
             else if (sm->endpoint_dependent)
               {
-                del_feature_name = !is_inside ?  "nat44-ed-in2out" :
+                del_feature_name = !is_inside ?  "nat-pre-in2out" :
-                                                 "nat44-ed-out2in";
+                                                 "nat-pre-out2in";
+
                 feature_name = "nat44-ed-classify";
               }
             else
@@ -2074,7 +2087,7 @@ feature_set:
     {
       if (sm->endpoint_dependent)
 	{
-	  vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-out2in",
+	  vnet_feature_enable_disable ("ip4-unicast", "nat-pre-out2in",
 				       sw_if_index, !is_del, 0, 0);
 	  vnet_feature_enable_disable ("ip4-output", "nat44-ed-in2out-output",
 				       sw_if_index, !is_del, 0, 0);
@@ -2091,11 +2104,11 @@ feature_set:
 fq:
   if (sm->fq_in2out_output_index == ~0 && sm->num_workers > 1)
     sm->fq_in2out_output_index =
-      vlib_frame_queue_main_init (sm->in2out_output_node_index, 0);
+      vlib_frame_queue_main_init (sm->handoff_in2out_output_index, 0);
 
   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
     sm->fq_out2in_index =
-      vlib_frame_queue_main_init (sm->out2in_node_index, 0);
+      vlib_frame_queue_main_init (sm->handoff_out2in_index, 0);
 
   /* *INDENT-OFF* */
   pool_foreach (i, sm->output_feature_interfaces,
@@ -2316,6 +2329,19 @@ snat_init (vlib_main_t * vm)
   node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
   sm->error_node_index = node->index;
 
+  node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
+  sm->pre_in2out_node_index = node->index;
+  node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
+  sm->pre_out2in_node_index = node->index;
+
+  node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-in2out");
+  sm->pre_in2out_node_index = node->index;
+
+  node = vlib_get_node_by_name (vm, (u8 *) "nat-pre-out2in");
+  sm->pre_out2in_node_index = node->index;
+
+  // TODO: output ?? (special node)
+
   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out");
   sm->in2out_node_index = node->index;
   node = vlib_get_node_by_name (vm, (u8 *) "nat44-in2out-output");
@@ -3839,9 +3865,16 @@ snat_config (vlib_main_t * vm, unformat_input_t * input)
 	{
 	  sm->worker_in2out_cb = nat44_ed_get_worker_in2out_cb;
 	  sm->worker_out2in_cb = nat44_ed_get_worker_out2in_cb;
+
+	  sm->handoff_out2in_index = nat_pre_out2in_node.index;
+	  sm->handoff_in2out_index = nat_pre_in2out_node.index;
+	  // TODO: test
+	  sm->handoff_in2out_output_index = nat44_ed_in2out_output_node.index;
+
 	  sm->in2out_node_index = nat44_ed_in2out_node.index;
 	  sm->in2out_output_node_index = nat44_ed_in2out_output_node.index;
 	  sm->out2in_node_index = nat44_ed_out2in_node.index;
+
 	  sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
 	  sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
 	  nat_affinity_init (vm);
@@ -3852,6 +3885,12 @@ snat_config (vlib_main_t * vm, unformat_input_t * input)
 	{
 	  sm->worker_in2out_cb = snat_get_worker_in2out_cb;
 	  sm->worker_out2in_cb = snat_get_worker_out2in_cb;
+
+	  sm->handoff_out2in_index = snat_in2out_node.index;
+	  sm->handoff_in2out_index = snat_out2in_node.index;
+	  // TODO: test
+	  sm->handoff_in2out_output_index = snat_in2out_output_node.index;
+
 	  sm->in2out_node_index = snat_in2out_node.index;
 	  sm->in2out_output_node_index = snat_in2out_output_node.index;
 	  sm->out2in_node_index = snat_out2in_node.index;
@@ -4263,6 +4302,38 @@ nat_set_alloc_addr_and_port_default (void)
   sm->alloc_addr_and_port = nat_alloc_addr_and_port_default;
 }
 
+VLIB_NODE_FN (nat_default_node) (vlib_main_t * vm,
+				 vlib_node_runtime_t * node,
+				 vlib_frame_t * frame)
+{
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (nat_default_node) = {
+  .name = "nat-default",
+  .vector_size = sizeof (u32),
+  .format_trace = 0,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = 0,
+  .n_next_nodes = NAT_N_NEXT,
+  .next_nodes = {
+    [NAT_NEXT_DROP] = "error-drop",
+    [NAT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+    [NAT_NEXT_IN2OUT_PRE] = "nat-pre-in2out",
+    [NAT_NEXT_OUT2IN_PRE] = "nat-pre-out2in",
+    [NAT_NEXT_IN2OUT_ED_FAST_PATH] = "nat44-ed-in2out",
+    [NAT_NEXT_IN2OUT_ED_SLOW_PATH] = "nat44-ed-in2out-slowpath",
+    [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
+    [NAT_NEXT_IN2OUT_ED_REASS] = "nat44-ed-in2out-reass",
+    [NAT_NEXT_IN2OUT_ED_OUTPUT_REASS] = "nat44-ed-in2out-reass-output",
+    [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
+    [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
+    [NAT_NEXT_OUT2IN_ED_REASS] = "nat44-ed-out2in-reass",
+  },
+};
+/* *INDENT-ON* */
+
 /*
  * fd.io coding-style-patch-verification: ON
  *

src/plugins/nat/nat.h

@@ -43,6 +43,45 @@
 /* NAT buffer flags */
 #define SNAT_FLAG_HAIRPINNING (1 << 0)
 
+typedef struct
+{
+  u32 arc_next;
+} nat_buffer_opaque_t;
+
+typedef enum
+{
+  NAT_NEXT_DROP,
+  NAT_NEXT_ICMP_ERROR,
+  NAT_NEXT_IN2OUT_PRE,
+  NAT_NEXT_OUT2IN_PRE,
+  NAT_NEXT_IN2OUT_ED_FAST_PATH,
+  NAT_NEXT_IN2OUT_ED_SLOW_PATH,
+  NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH,
+  NAT_NEXT_IN2OUT_ED_REASS,
+  NAT_NEXT_IN2OUT_ED_OUTPUT_REASS,
+  NAT_NEXT_OUT2IN_ED_FAST_PATH,
+  NAT_NEXT_OUT2IN_ED_SLOW_PATH,
+  NAT_NEXT_OUT2IN_ED_REASS,
+  NAT_N_NEXT,
+} nat_next_t;
+
+typedef struct
+{
+  u32 next_index;
+} nat_pre_trace_t;
+
+#define nat_buffer_opaque(b) \
+  ((nat_buffer_opaque_t *)((vnet_buffer_opaque2_t *)b->opaque2)->__unused2)
+
+/*
+STATIC_ASSERT (sizeof (nat_buffer_opaque_t) <=
+               STRUCT_SIZE_OF (vnet_buffer_opaque_t, unused),
+               "Custom meta-data too large for vnet_buffer_opaque_t");
+
+#define nat_buffer_opaque(b) \
+  ((nat_buffer_opaque_t *)((u8 *)((b)->opaque) + \
+    STRUCT_OFFSET_OF (vnet_buffer_opaque_t, unused)))*/
+
 /* session key (4-tuple) */
 typedef struct
 {
@@ -575,6 +614,15 @@ typedef struct snat_main_s
   /* node indexes */
   u32 error_node_index;
 
+  /* handoff fq nodes  */
+  u32 handoff_out2in_index;
+  u32 handoff_in2out_index;
+  u32 handoff_in2out_output_index;
+
+  /* respect feature arc nodes */
+  u32 pre_out2in_node_index;
+  u32 pre_in2out_node_index;
+
   u32 in2out_node_index;
   u32 in2out_output_node_index;
   u32 in2out_fast_node_index;
@@ -667,6 +715,12 @@ typedef struct
 } snat_runtime_t;
 
 extern snat_main_t snat_main;
+
+// nat pre ed next_node feature classification
+extern vlib_node_registration_t nat_default_node;
+extern vlib_node_registration_t nat_pre_in2out_node;
+extern vlib_node_registration_t nat_pre_out2in_node;
+
 extern vlib_node_registration_t snat_in2out_node;
 extern vlib_node_registration_t snat_in2out_output_node;
 extern vlib_node_registration_t snat_out2in_node;

src/plugins/nat/nat44_handoff.c

@@ -23,6 +23,7 @@
 #include <vnet/fib/ip4_fib.h>
 #include <vppinfra/error.h>
 #include <nat/nat.h>
+#include <nat/nat_inlines.h>
 
 typedef struct
 {
@@ -260,14 +261,11 @@ VLIB_NODE_FN (snat_in2out_worker_handoff_node) (vlib_main_t * vm,
 VLIB_REGISTER_NODE (snat_in2out_worker_handoff_node) = {
   .name = "nat44-in2out-worker-handoff",
   .vector_size = sizeof (u32),
+  .sibling_of = "nat-default",
   .format_trace = format_nat44_handoff_trace,
   .type = VLIB_NODE_TYPE_INTERNAL,
   .n_errors = ARRAY_LEN(nat44_handoff_error_strings),
   .error_strings = nat44_handoff_error_strings,
-  .n_next_nodes = 1,
-  .next_nodes = {
-    [0] = "error-drop",
-  },
 };
 /* *INDENT-ON* */
 
@@ -283,14 +281,11 @@ VLIB_NODE_FN (snat_in2out_output_worker_handoff_node) (vlib_main_t * vm,
 VLIB_REGISTER_NODE (snat_in2out_output_worker_handoff_node) = {
   .name = "nat44-in2out-output-worker-handoff",
   .vector_size = sizeof (u32),
+  .sibling_of = "nat-default",
   .format_trace = format_nat44_handoff_trace,
   .type = VLIB_NODE_TYPE_INTERNAL,
   .n_errors = ARRAY_LEN(nat44_handoff_error_strings),
   .error_strings = nat44_handoff_error_strings,
-  .n_next_nodes = 1,
-  .next_nodes = {
-    [0] = "error-drop",
-  },
 };
 /* *INDENT-ON* */
 
@@ -305,14 +300,11 @@ VLIB_NODE_FN (snat_out2in_worker_handoff_node) (vlib_main_t * vm,
 VLIB_REGISTER_NODE (snat_out2in_worker_handoff_node) = {
   .name = "nat44-out2in-worker-handoff",
   .vector_size = sizeof (u32),
+  .sibling_of = "nat-default",
   .format_trace = format_nat44_handoff_trace,
   .type = VLIB_NODE_TYPE_INTERNAL,
   .n_errors = ARRAY_LEN(nat44_handoff_error_strings),
   .error_strings = nat44_handoff_error_strings,
-  .n_next_nodes = 1,
-  .next_nodes = {
-    [0] = "error-drop",
-  },
 };
 /* *INDENT-ON* */
 

src/plugins/nat/nat_inlines.h

@@ -23,6 +23,126 @@
 #include <nat/nat.h>
 #include <nat/nat_ha.h>
 
+static inline uword
+nat_pre_node_fn_inline (vlib_main_t * vm,
+			vlib_node_runtime_t * node,
+			vlib_frame_t * frame, u32 def_next)
+{
+  u32 n_left_from, *from, *to_next;
+  u16 next_index;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+  next_index = node->cached_next_index;
+
+  while (n_left_from > 0)
+    {
+      u32 n_left_to_next;
+
+      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+      while (n_left_from >= 4 && n_left_to_next >= 2)
+	{
+	  u32 next0, next1;
+	  u32 arc_next0, arc_next1;
+	  u32 bi0, bi1;
+	  vlib_buffer_t *b0, *b1;
+
+	  /* Prefetch next iteration. */
+	  {
+	    vlib_buffer_t *p2, *p3;
+
+	    p2 = vlib_get_buffer (vm, from[2]);
+	    p3 = vlib_get_buffer (vm, from[3]);
+
+	    vlib_prefetch_buffer_header (p2, LOAD);
+	    vlib_prefetch_buffer_header (p3, LOAD);
+
+	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
+	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  }
+
+	  /* speculatively enqueue b0 and b1 to the current next frame */
+	  to_next[0] = bi0 = from[0];
+	  to_next[1] = bi1 = from[1];
+	  from += 2;
+	  to_next += 2;
+	  n_left_from -= 2;
+	  n_left_to_next -= 2;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  b1 = vlib_get_buffer (vm, bi1);
+
+	  next0 = def_next;
+	  next1 = def_next;
+
+	  vnet_feature_next (&arc_next0, b0);
+	  vnet_feature_next (&arc_next1, b1);
+
+	  nat_buffer_opaque (b0)->arc_next = arc_next0;
+	  nat_buffer_opaque (b1)->arc_next = arc_next1;
+
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	    {
+	      if (b0->flags & VLIB_BUFFER_IS_TRACED)
+		{
+		  nat_pre_trace_t *t =
+		    vlib_add_trace (vm, node, b0, sizeof (*t));
+		  t->next_index = next0;
+		}
+	      if (b1->flags & VLIB_BUFFER_IS_TRACED)
+		{
+		  nat_pre_trace_t *t =
+		    vlib_add_trace (vm, node, b0, sizeof (*t));
+		  t->next_index = next0;
+		}
+	    }
+
+	  /* verify speculative enqueues, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, bi1, next0, next1);
+	}
+
+      while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	  u32 next0;
+	  u32 arc_next0;
+	  u32 bi0;
+	  vlib_buffer_t *b0;
+
+	  /* speculatively enqueue b0 to the current next frame */
+	  bi0 = from[0];
+	  to_next[0] = bi0;
+	  from += 1;
+	  to_next += 1;
+	  n_left_from -= 1;
+	  n_left_to_next -= 1;
+
+	  b0 = vlib_get_buffer (vm, bi0);
+	  next0 = def_next;
+	  vnet_feature_next (&arc_next0, b0);
+	  nat_buffer_opaque (b0)->arc_next = arc_next0;
+
+	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
+			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+	    {
+	      nat_pre_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
+	      t->next_index = next0;
+	    }
+
+	  /* verify speculative enqueue, maybe switch current next frame */
+	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+					   to_next, n_left_to_next,
+					   bi0, next0);
+	}
+
+      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+  return frame->n_vectors;
+}
+
 always_inline u32
 ip_proto_to_snat_proto (u8 ip_proto)
 {
@@ -552,13 +672,13 @@ snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t * node,
 	return 1;
 
       snat_interface_t *i;
-      pool_foreach (i, sm->interfaces, (
+      /* *INDENT-OFF* */
-					 {
+      pool_foreach (i, sm->interfaces, ({
-					 /* NAT packet aimed at outside interface */
+        /* NAT packet aimed at outside interface */
-					 if ((nat_interface_is_outside (i))
+	if ((nat_interface_is_outside (i)) && (sw_if_index == i->sw_if_index))
-					     && (sw_if_index ==
+          return 0;
-						 i->sw_if_index)) return 0;}
+      }));
-		    ));
+      /* *INDENT-ON* */
     }
 
   return 1;