ipsec: fix AES CBC IV generation (CVE-2022-46397)

For AES-CBC, the IV must be unpredictable (see NIST SP800-38a Appendix
C). Chaining IVs like is done by ipsecmb and native backends for the
VNET_CRYPTO_OP_FLAG_INIT_IV is fully predictable.
Encrypt a counter as part of the message, making the (predictable)
counter-generated IV unpredictable.

Fixes: VPP-2037
Type: fix

Change-Id: If4f192d62bf97dda553e7573331c75efa11822ae
Signed-off-by: Benoît Ganne <bganne@cisco.com>

.gitreview

@@ -2,3 +2,4 @@
 host=gerrit.fd.io
 port=29418
 project=vpp
+defaultbranch=stable/2202

MAINTAINERS

@@ -42,6 +42,7 @@ Sphinx Documents
 I:	docs
 M:	John DeNisco <jdenisco@cisco.com>
 M:	Ray Kinsella <raykinsella78@gmail.com>
+M:	Dave Wallace <dwallacelf@gmail.com>
 F:	docs/
 
 Infrastructure Library

docs/aboutvpp/releasenotes/index.rst

@@ -6,6 +6,7 @@ Release notes
 .. toctree::
     :maxdepth: 2
 
+    v22.02
     v21.10.1
     v21.10
     v21.06

docs/usecases/vpp_testbench/index.rst

@@ -10,7 +10,7 @@ back-and-forth (i.e. ICMP echo/ping requests and HTTP GET requests).
 The intent of this example is to provide a relatively simple example of
 connecting containers via VPP and allowing others to use it as a springboard of
 sorts for their own projects and examples. Besides Docker and a handful of
-common Linux command-line utlities, not much else is required to build this
+common Linux command-line utilities, not much else is required to build this
 example (due to most of the dependencies being lumped inside the containers
 themselves).
 
@@ -60,7 +60,7 @@ project.
   other scripts in this project. Intended to be sourced (i.e. not intended to
   be run directly). Some of the helper functions are used at run-time within
   the containers, while others are intended to be run in the default namespace
-  on the host operating system to help with run-time configuration/bringup of
+  on the host operating system to help with run-time configuration/bring up of
   the testbench.
 * ``Dockerfile.vpp_testbench``: used to build the various Docker images used in
   this project (i.e. so VPP, our test tools, etc.; are all encapsulated within
@@ -81,7 +81,7 @@ Getting Started
 ^^^^^^^^^^^^^^^
 
 First, we'll assume you are running on a Ubuntu 20.04 x86_64 setup (either on a
-bare metal host or in a virtual machine), and have acquirec a copy of the
+bare metal host or in a virtual machine), and have acquired a copy of the
 project files (either by cloning the VPP git repository, or duplicating them
 from :ref:`sec_file_listings_vpp_testbench`). Now, just run ``make``. The
 process should take a few minutes as it pulls the baseline Ubuntu Docker image,
@@ -96,11 +96,11 @@ can be cleaned-up via ``make stop``, and the whole process of starting,
 testing, stopping, etc.; can be repeated as needed.
 
 In addition to starting up the containers, ``make start`` will establish
-variaous types of links/connections between the two containers, making use of
+various types of links/connections between the two containers, making use of
 both the Linux network stack, as well as VPP, to handle the "plumbing"
 involved. This is to allow various types of connections between the two
 containers, and to allow the reader to experiment with them (i.e. using
-``vppctl`` to congfigure or trace packets going over VPP-managed links, use
+``vppctl`` to configure or trace packets going over VPP-managed links, use
 traditional Linux command line utilities like ``tcpdump``, ``iproute2``,
 ``ping``, etc.; to accomplish similar tasks over links managed purely by the
 Linux network stack, etc.). Later labs will also encourage readers to compare
@@ -177,4 +177,3 @@ entrypoint_server.sh
    :caption: entrypoint_server.sh
    :language: shell
    :linenos:
-

src/plugins/dpdk/cryptodev/cryptodev.c

@@ -51,12 +51,19 @@ prepare_aead_xform (struct rte_crypto_sym_xform *xform,
   xform->type = RTE_CRYPTO_SYM_XFORM_AEAD;
   xform->next = 0;
 
-  if (key->alg != VNET_CRYPTO_ALG_AES_128_GCM &&
-      key->alg != VNET_CRYPTO_ALG_AES_192_GCM &&
-      key->alg != VNET_CRYPTO_ALG_AES_256_GCM)
+  if (key->alg == VNET_CRYPTO_ALG_AES_128_GCM ||
+      key->alg == VNET_CRYPTO_ALG_AES_192_GCM ||
+      key->alg == VNET_CRYPTO_ALG_AES_256_GCM)
+    {
+      aead_xform->algo = RTE_CRYPTO_AEAD_AES_GCM;
+    }
+  else if (key->alg == VNET_CRYPTO_ALG_CHACHA20_POLY1305)
+    {
+      aead_xform->algo = RTE_CRYPTO_AEAD_CHACHA20_POLY1305;
+    }
+  else
     return -1;
 
-  aead_xform->algo = RTE_CRYPTO_AEAD_AES_GCM;
   aead_xform->op = (op_type == CRYPTODEV_OP_TYPE_ENCRYPT) ?
     RTE_CRYPTO_AEAD_OP_ENCRYPT : RTE_CRYPTO_AEAD_OP_DECRYPT;
   aead_xform->aad_length = aad_len;

src/plugins/dpdk/cryptodev/cryptodev.h

@@ -43,7 +43,10 @@
   _ (AES_192_GCM, AEAD, AES_GCM, 12, 16, 8, 24)                               \
   _ (AES_192_GCM, AEAD, AES_GCM, 12, 16, 12, 24)                              \
   _ (AES_256_GCM, AEAD, AES_GCM, 12, 16, 8, 32)                               \
-  _ (AES_256_GCM, AEAD, AES_GCM, 12, 16, 12, 32)
+  _ (AES_256_GCM, AEAD, AES_GCM, 12, 16, 12, 32)                              \
+  _ (CHACHA20_POLY1305, AEAD, CHACHA20_POLY1305, 12, 16, 0, 32)               \
+  _ (CHACHA20_POLY1305, AEAD, CHACHA20_POLY1305, 12, 16, 8, 32)               \
+  _ (CHACHA20_POLY1305, AEAD, CHACHA20_POLY1305, 12, 16, 12, 32)
 
 /**
  * crypto (alg, cryptodev_alg, key_size), hash (alg, digest-size)

src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c

@@ -458,6 +458,13 @@ cryptodev_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed,
   return frame;
 }
 
+static_always_inline int
+cryptodev_enqueue_aead_aad_0_enc (vlib_main_t *vm,
+				  vnet_crypto_async_frame_t *frame)
+{
+  return cryptodev_frame_aead_enqueue (vm, frame, CRYPTODEV_OP_TYPE_ENCRYPT,
+				       0);
+}
 static_always_inline int
 cryptodev_enqueue_aead_aad_8_enc (vlib_main_t *vm,
 				  vnet_crypto_async_frame_t *frame)
@@ -473,6 +480,13 @@ cryptodev_enqueue_aead_aad_12_enc (vlib_main_t *vm,
 				       12);
 }
 
+static_always_inline int
+cryptodev_enqueue_aead_aad_0_dec (vlib_main_t *vm,
+				  vnet_crypto_async_frame_t *frame)
+{
+  return cryptodev_frame_aead_enqueue (vm, frame, CRYPTODEV_OP_TYPE_DECRYPT,
+				       0);
+}
 static_always_inline int
 cryptodev_enqueue_aead_aad_8_dec (vlib_main_t *vm,
 				  vnet_crypto_async_frame_t *frame)

src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c

@@ -349,7 +349,7 @@ cryptodev_raw_aead_enqueue (vlib_main_t *vm, vnet_crypto_async_frame_t *frame,
 
       if (aad_len == 8)
 	*(u64 *) (cet->aad_buf + aad_offset) = *(u64 *) fe->aad;
-      else
+      else if (aad_len != 0)
 	{
 	  /* aad_len == 12 */
 	  *(u64 *) (cet->aad_buf + aad_offset) = *(u64 *) fe->aad;
@@ -591,6 +591,13 @@ end_deq:
   return frame_ret;
 }
 
+static_always_inline int
+cryptodev_raw_enq_aead_aad_0_enc (vlib_main_t *vm,
+				  vnet_crypto_async_frame_t *frame)
+{
+  return cryptodev_raw_aead_enqueue (vm, frame, CRYPTODEV_OP_TYPE_ENCRYPT, 0);
+}
+
 static_always_inline int
 cryptodev_raw_enq_aead_aad_8_enc (vlib_main_t *vm,
 				  vnet_crypto_async_frame_t *frame)
@@ -604,6 +611,13 @@ cryptodev_raw_enq_aead_aad_12_enc (vlib_main_t *vm,
   return cryptodev_raw_aead_enqueue (vm, frame, CRYPTODEV_OP_TYPE_ENCRYPT, 12);
 }
 
+static_always_inline int
+cryptodev_raw_enq_aead_aad_0_dec (vlib_main_t *vm,
+				  vnet_crypto_async_frame_t *frame)
+{
+  return cryptodev_raw_aead_enqueue (vm, frame, CRYPTODEV_OP_TYPE_DECRYPT, 0);
+}
+
 static_always_inline int
 cryptodev_raw_enq_aead_aad_8_dec (vlib_main_t *vm,
 				  vnet_crypto_async_frame_t *frame)

src/plugins/dpdk/device/driver.c

@@ -55,11 +55,16 @@ static dpdk_driver_t dpdk_drivers[] = {
     .int_unmaskable = 1,
   },
   {
-    .drivers = DPDK_DRIVERS ({ "net_e1000_igb_vf", "Intel e1000 VF" },
-			     { "net_ixgbe_vf", "Intel 82599 VF" }),
+    .drivers = DPDK_DRIVERS ({ "net_e1000_igb_vf", "Intel e1000 VF" }),
     .interface_name_prefix = "VirtualFunctionEthernet",
     .use_intel_phdr_cksum = 1,
   },
+  {
+    .drivers = DPDK_DRIVERS ({ "net_ixgbe_vf", "Intel 82599 VF" }),
+    .interface_name_prefix = "VirtualFunctionEthernet",
+    .use_intel_phdr_cksum = 1,
+    .program_vlans = 1,
+  },
   {
     .drivers = DPDK_DRIVERS ({ "net_dpaa2", "NXP DPAA2 Mac" }),
     .interface_name_prefix = "TenGigabitEthernet",
@@ -108,7 +113,6 @@ static dpdk_driver_t dpdk_drivers[] = {
     .drivers = DPDK_DRIVERS ({ "net_ena", "AWS ENA VF" }),
     .interface_name_prefix = "VirtualFunctionEthernet",
     .enable_rxq_int = 1,
-    .disable_rx_scatter = 1,
   },
   {
     .drivers = DPDK_DRIVERS ({ "net_vmxnet3", "VMware VMXNET3" }),

src/plugins/dpdk/device/node.c

@@ -364,12 +364,13 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd,
   /* get up to DPDK_RX_BURST_SZ buffers from PMD */
   while (n_rx_packets < DPDK_RX_BURST_SZ)
     {
-      n = rte_eth_rx_burst (xd->port_id, queue_id,
-			    ptd->mbufs + n_rx_packets,
-			    DPDK_RX_BURST_SZ - n_rx_packets);
+      u32 n_to_rx = clib_min (DPDK_RX_BURST_SZ - n_rx_packets, 32);
+
+      n = rte_eth_rx_burst (xd->port_id, queue_id, ptd->mbufs + n_rx_packets,
+			    n_to_rx);
       n_rx_packets += n;
 
-      if (n < 32)
+      if (n < n_to_rx)
 	break;
     }
 

src/plugins/lisp/lisp-gpe/interface.c

@@ -92,7 +92,6 @@ VLIB_NODE_FN (lisp_tunnel_output)
 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
 {
   u32 n_left_from, next_index, *from, *to_next;
-  lisp_gpe_main_t *lgm = &lisp_gpe_main;
 
   from = vlib_frame_vector_args (from_frame);
   n_left_from = from_frame->n_vectors;
@@ -111,7 +110,6 @@ VLIB_NODE_FN (lisp_tunnel_output)
 	  const ip_adjacency_t *adj0;
 	  const dpo_id_t *dpo0;
 	  vlib_buffer_t *b0;
-	  u8 is_v4_0;
 
 	  bi0 = from[0];
 	  to_next[0] = bi0;
@@ -123,11 +121,6 @@ VLIB_NODE_FN (lisp_tunnel_output)
 	  b0 = vlib_get_buffer (vm, bi0);
 	  b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
 
-	  /* Fixup the checksum and len fields in the LISP tunnel encap
-	   * that was applied at the midchain node */
-	  is_v4_0 = is_v4_packet (vlib_buffer_get_current (b0));
-	  ip_udp_fixup_one (lgm->vlib_main, b0, is_v4_0);
-
 	  /* Follow the DPO on which the midchain is stacked */
 	  adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
 	  adj0 = adj_get (adj_index0);

src/plugins/memif/memif.c

@@ -50,6 +50,14 @@ memif_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags)
   return 0;
 }
 
+static clib_error_t *
+memif_eth_set_max_frame_size (vnet_main_t *vnm, vnet_hw_interface_t *hi,
+			      u32 flags)
+{
+  /* nothing for now */
+  return 0;
+}
+
 static void
 memif_queue_intfd_close (memif_queue_t * mq)
 {
@@ -1017,6 +1025,7 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args)
       eir.dev_instance = mif->dev_instance;
       eir.address = args->hw_addr;
       eir.cb.flag_change = memif_eth_flag_change;
+      eir.cb.set_max_frame_size = memif_eth_set_max_frame_size;
       mif->hw_if_index = vnet_eth_register_interface (vnm, &eir);
     }
   else if (mif->mode == MEMIF_INTERFACE_MODE_IP)

src/plugins/memif/node.c

@@ -247,7 +247,7 @@ memif_process_desc (vlib_main_t *vm, vlib_node_runtime_t *node,
   u32 n_buffers = 0;
   u32 n_left = ptd->n_packets;
   u32 packet_len;
-  int i = 0;
+  int i = -1;
 
   /* construct copy and packet vector out of ring slots */
   while (n_left)
@@ -261,6 +261,7 @@ memif_process_desc (vlib_main_t *vm, vlib_node_runtime_t *node,
       dst_off = start_offset;
 
     next_slot:
+      i++; /* next descriptor */
       n_bytes_left = desc_len[i];
 
       packet_len += n_bytes_left;
@@ -287,9 +288,6 @@ memif_process_desc (vlib_main_t *vm, vlib_node_runtime_t *node,
 	  }
 	while (PREDICT_FALSE (n_bytes_left));
 
-      /* next descriptor */
-      i++;
-
       if (desc_status[i].next)
 	{
 	  src_off = 0;
@@ -557,7 +555,7 @@ memif_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
 
       vlib_get_buffers (vm, ptd->buffers, buffer_ptrs, n_buffers);
 
-      for (i = 0; i < n_pkts - 8; i++)
+      for (i = 0; i + 8 < n_pkts; i++)
 	{
 	  clib_prefetch_load (desc_data[i + 8]);
 	  clib_prefetch_store (buffer_ptrs[i + 8]->data);
@@ -574,7 +572,7 @@ memif_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
       u32 n_pkts = vec_len (ptd->copy_ops);
       co = ptd->copy_ops;
 
-      for (i = 0; i < n_pkts - 8; i++)
+      for (i = 0; i + 8 < n_pkts; i++)
 	{
 	  clib_prefetch_load (co[i + 8].data);
 	  b = vlib_get_buffer (vm, ptd->buffers[co[i].buffer_vec_index]);

src/plugins/wireguard/wireguard.api

@@ -195,6 +195,17 @@ define wireguard_peers_details {
   vl_api_wireguard_peer_t peer;
 };
 
+/** \brief Wireguard Set Async mode
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param async_enable - wireguard async mode on or off
+*/
+autoreply define wg_set_async_mode {
+  u32 client_index;
+  u32 context;
+  bool async_enable;
+};
+
 /*
  * Local Variables:
  * eval: (c-set-style "gnu")

src/plugins/wireguard/wireguard.c

@@ -16,6 +16,7 @@
 #include <vnet/vnet.h>
 #include <vnet/plugin/plugin.h>
 #include <vpp/app/version.h>
+#include <vnet/crypto/crypto.h>
 
 #include <wireguard/wireguard_send.h>
 #include <wireguard/wireguard_key.h>
@@ -23,6 +24,40 @@
 #include <wireguard/wireguard.h>
 
 wg_main_t wg_main;
+wg_async_post_next_t wg_encrypt_async_next;
+wg_async_post_next_t wg_decrypt_async_next;
+
+void
+wg_set_async_mode (u32 is_enabled)
+{
+  vnet_crypto_request_async_mode (is_enabled);
+
+  if (is_enabled)
+    wg_op_mode_set_ASYNC ();
+  else
+    wg_op_mode_unset_ASYNC ();
+}
+
+static void
+wireguard_register_post_node (vlib_main_t *vm)
+
+{
+  wg_async_post_next_t *eit;
+  wg_async_post_next_t *dit;
+
+  eit = &wg_encrypt_async_next;
+  dit = &wg_decrypt_async_next;
+
+  eit->wg4_post_next =
+    vnet_crypto_register_post_node (vm, "wg4-output-tun-post-node");
+  eit->wg6_post_next =
+    vnet_crypto_register_post_node (vm, "wg6-output-tun-post-node");
+
+  dit->wg4_post_next =
+    vnet_crypto_register_post_node (vm, "wg4-input-post-node");
+  dit->wg6_post_next =
+    vnet_crypto_register_post_node (vm, "wg6-input-post-node");
+}
 
 static clib_error_t *
 wg_init (vlib_main_t * vm)
@@ -44,6 +79,8 @@ wg_init (vlib_main_t * vm)
 			CLIB_CACHE_LINE_BYTES);
 
   wg_timer_wheel_init ();
+  wireguard_register_post_node (vm);
+  wmp->op_mode_flags = 0;
 
   return (NULL);
 }

src/plugins/wireguard/wireguard.h

@@ -28,6 +28,9 @@ extern vlib_node_registration_t wg6_output_tun_node;
 
 typedef struct wg_per_thread_data_t_
 {
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  vnet_crypto_op_t *crypto_ops;
+  vnet_crypto_async_frame_t **async_frames;
   u8 data[WG_DEFAULT_DATA_SIZE];
 } wg_per_thread_data_t;
 typedef struct
@@ -48,12 +51,70 @@ typedef struct
   u8 feature_init;
 
   tw_timer_wheel_16t_2w_512sl_t timer_wheel;
+
+  /* operation mode flags (e.g. async) */
+  u8 op_mode_flags;
 } wg_main_t;
 
+typedef struct
+{
+  /* wg post node index for async crypto */
+  u32 wg4_post_next;
+  u32 wg6_post_next;
+} wg_async_post_next_t;
+
+extern wg_async_post_next_t wg_encrypt_async_next;
+extern wg_async_post_next_t wg_decrypt_async_next;
 extern wg_main_t wg_main;
 
+/**
+ * Wireguard operation mode
+ **/
+#define foreach_wg_op_mode_flags _ (0, ASYNC, "async")
+
+/**
+ * Helper function to set/unset and check op modes
+ **/
+typedef enum wg_op_mode_flags_t_
+{
+#define _(v, f, s) WG_OP_MODE_FLAG_##f = 1 << v,
+  foreach_wg_op_mode_flags
+#undef _
+} __clib_packed wg_op_mode_flags_t;
+
+#define _(a, v, s)                                                            \
+  always_inline int wg_op_mode_set_##v (void)                                 \
+  {                                                                           \
+    return (wg_main.op_mode_flags |= WG_OP_MODE_FLAG_##v);                    \
+  }                                                                           \
+  always_inline int wg_op_mode_unset_##v (void)                               \
+  {                                                                           \
+    return (wg_main.op_mode_flags &= ~WG_OP_MODE_FLAG_##v);                   \
+  }                                                                           \
+  always_inline int wg_op_mode_is_set_##v (void)                              \
+  {                                                                           \
+    return (wg_main.op_mode_flags & WG_OP_MODE_FLAG_##v);                     \
+  }
+foreach_wg_op_mode_flags
+#undef _
+
+  typedef struct
+{
+  u8 __pad[22];
+  u16 next_index;
+} wg_post_data_t;
+
+STATIC_ASSERT (sizeof (wg_post_data_t) <=
+		 STRUCT_SIZE_OF (vnet_buffer_opaque_t, unused),
+	       "Custom meta-data too large for vnet_buffer_opaque_t");
+
+#define wg_post_data(b)                                                       \
+  ((wg_post_data_t *) ((u8 *) ((b)->opaque) +                                 \
+		       STRUCT_OFFSET_OF (vnet_buffer_opaque_t, unused)))
+
 #define WG_START_EVENT	1
 void wg_feature_init (wg_main_t * wmp);
+void wg_set_async_mode (u32 is_enabled);
 
 #endif /* __included_wg_h__ */
 

src/plugins/wireguard/wireguard_api.c

@@ -365,6 +365,18 @@ wg_api_peer_event (index_t peeri, wg_peer_flags flags)
     };
 }
 
+static void
+vl_api_wg_set_async_mode_t_handler (vl_api_wg_set_async_mode_t *mp)
+{
+  wg_main_t *wmp = &wg_main;
+  vl_api_wg_set_async_mode_reply_t *rmp;
+  int rv = 0;
+
+  wg_set_async_mode (mp->async_enable);
+
+  REPLY_MACRO (VL_API_WG_SET_ASYNC_MODE_REPLY);
+}
+
 /* set tup the API message handling tables */
 #include <wireguard/wireguard.api.c>
 static clib_error_t *

src/plugins/wireguard/wireguard_cli.c

@@ -356,6 +356,61 @@ VLIB_CLI_COMMAND (wg_show_itfs_command, static) =
   .short_help = "show wireguard",
   .function = wg_show_if_command_fn,
 };
+
+static clib_error_t *
+wg_set_async_mode_command_fn (vlib_main_t *vm, unformat_input_t *input,
+			      vlib_cli_command_t *cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  int async_enable = 0;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "on"))
+	async_enable = 1;
+      else if (unformat (line_input, "off"))
+	async_enable = 0;
+      else
+	return (clib_error_return (0, "unknown input '%U'",
+				   format_unformat_error, line_input));
+    }
+
+  wg_set_async_mode (async_enable);
+
+  unformat_free (line_input);
+  return (NULL);
+}
+
+VLIB_CLI_COMMAND (wg_set_async_mode_command, static) = {
+  .path = "set wireguard async mode",
+  .short_help = "set wireguard async mode on|off",
+  .function = wg_set_async_mode_command_fn,
+};
+
+static clib_error_t *
+wg_show_mode_command_fn (vlib_main_t *vm, unformat_input_t *input,
+			 vlib_cli_command_t *cmd)
+{
+  vlib_cli_output (vm, "Wireguard mode");
+
+#define _(v, f, s)                                                            \
+  vlib_cli_output (vm, "\t%s: %s", s,                                         \
+		   (wg_op_mode_is_set_##f () ? "enabled" : "disabled"));
+  foreach_wg_op_mode_flags
+#undef _
+
+    return (NULL);
+}
+
+VLIB_CLI_COMMAND (wg_show_modemode_command, static) = {
+  .path = "show wireguard mode",
+  .short_help = "show wireguard mode",
+  .function = wg_show_mode_command_fn,
+};
+
 /* *INDENT-ON* */
 
 /*

src/plugins/wireguard/wireguard_noise.c

@@ -36,7 +36,7 @@ static uint32_t noise_remote_handshake_index_get (noise_remote_t *);
 static void noise_remote_handshake_index_drop (noise_remote_t *);
 
 static uint64_t noise_counter_send (noise_counter_t *);
-static bool noise_counter_recv (noise_counter_t *, uint64_t);
+bool noise_counter_recv (noise_counter_t *, uint64_t);
 
 static void noise_kdf (uint8_t *, uint8_t *, uint8_t *, const uint8_t *,
 		       size_t, size_t, size_t, size_t,
@@ -407,6 +407,8 @@ noise_remote_begin_session (vlib_main_t * vm, noise_remote_t * r)
 
   /* Now we need to add_new_keypair */
   clib_rwlock_writer_lock (&r->r_keypair_lock);
+  /* Activate barrier to synchronization keys between threads */
+  vlib_worker_thread_barrier_sync (vm);
   next = r->r_next;
   current = r->r_current;
   previous = r->r_previous;
@@ -438,6 +440,7 @@ noise_remote_begin_session (vlib_main_t * vm, noise_remote_t * r)
       r->r_next = noise_remote_keypair_allocate (r);
       *r->r_next = kp;
     }
+  vlib_worker_thread_barrier_release (vm);
   clib_rwlock_writer_unlock (&r->r_keypair_lock);
 
   secure_zero_memory (&r->r_handshake, sizeof (r->r_handshake));
@@ -591,88 +594,6 @@ error:
   return ret;
 }
 
-enum noise_state_crypt
-noise_remote_decrypt (vlib_main_t * vm, noise_remote_t * r, uint32_t r_idx,
-		      uint64_t nonce, uint8_t * src, size_t srclen,
-		      uint8_t * dst)
-{
-  noise_keypair_t *kp;
-  enum noise_state_crypt ret = SC_FAILED;
-
-  if (r->r_current != NULL && r->r_current->kp_local_index == r_idx)
-    {
-      kp = r->r_current;
-    }
-  else if (r->r_previous != NULL && r->r_previous->kp_local_index == r_idx)
-    {
-      kp = r->r_previous;
-    }
-  else if (r->r_next != NULL && r->r_next->kp_local_index == r_idx)
-    {
-      kp = r->r_next;
-    }
-  else
-    {
-      goto error;
-    }
-
-  /* We confirm that our values are within our tolerances. These values
-   * are the same as the encrypt routine.
-   *
-   * kp_ctr isn't locked here, we're happy to accept a racy read. */
-  if (wg_birthdate_has_expired (kp->kp_birthdate, REJECT_AFTER_TIME) ||
-      kp->kp_ctr.c_recv >= REJECT_AFTER_MESSAGES)
-    goto error;
-
-  /* Decrypt, then validate the counter. We don't want to validate the
-   * counter before decrypting as we do not know the message is authentic
-   * prior to decryption. */
-  if (!chacha20poly1305_calc (vm, src, srclen, dst, NULL, 0, nonce,
-			      VNET_CRYPTO_OP_CHACHA20_POLY1305_DEC,
-			      kp->kp_recv_index))
-    goto error;
-
-  if (!noise_counter_recv (&kp->kp_ctr, nonce))
-    goto error;
-
-  /* If we've received the handshake confirming data packet then move the
-   * next keypair into current. If we do slide the next keypair in, then
-   * we skip the REKEY_AFTER_TIME_RECV check. This is safe to do as a
-   * data packet can't confirm a session that we are an INITIATOR of. */
-  if (kp == r->r_next)
-    {
-      clib_rwlock_writer_lock (&r->r_keypair_lock);
-      if (kp == r->r_next && kp->kp_local_index == r_idx)
-	{
-	  noise_remote_keypair_free (vm, r, &r->r_previous);
-	  r->r_previous = r->r_current;
-	  r->r_current = r->r_next;
-	  r->r_next = NULL;
-
-	  ret = SC_CONN_RESET;
-	  clib_rwlock_writer_unlock (&r->r_keypair_lock);
-	  goto error;
-	}
-      clib_rwlock_writer_unlock (&r->r_keypair_lock);
-    }
-
-  /* Similar to when we encrypt, we want to notify the caller when we
-   * are approaching our tolerances. We notify if:
-   *  - we're the initiator and the current keypair is older than
-   *    REKEY_AFTER_TIME_RECV seconds. */
-  ret = SC_KEEP_KEY_FRESH;
-  kp = r->r_current;
-  if (kp != NULL &&
-      kp->kp_valid &&
-      kp->kp_is_initiator &&
-      wg_birthdate_has_expired (kp->kp_birthdate, REKEY_AFTER_TIME_RECV))
-    goto error;
-
-  ret = SC_OK;
-error:
-  return ret;
-}
-
 /* Private functions - these should not be called outside this file under any
  * circumstances. */
 static noise_keypair_t *
@@ -683,21 +604,6 @@ noise_remote_keypair_allocate (noise_remote_t * r)
   return kp;
 }
 
-static void
-noise_remote_keypair_free (vlib_main_t * vm, noise_remote_t * r,
-			   noise_keypair_t ** kp)
-{
-  noise_local_t *local = noise_local_get (r->r_local_idx);
-  struct noise_upcall *u = &local->l_upcall;
-  if (*kp)
-    {
-      u->u_index_drop ((*kp)->kp_local_index);
-      vnet_crypto_key_del (vm, (*kp)->kp_send_index);
-      vnet_crypto_key_del (vm, (*kp)->kp_recv_index);
-      clib_mem_free (*kp);
-    }
-}
-
 static uint32_t
 noise_remote_handshake_index_get (noise_remote_t * r)
 {
@@ -716,55 +622,6 @@ noise_remote_handshake_index_drop (noise_remote_t * r)
     u->u_index_drop (hs->hs_local_index);
 }
 
-static uint64_t
-noise_counter_send (noise_counter_t * ctr)
-{
-  uint64_t ret;
-  ret = ctr->c_send++;
-  return ret;
-}
-
-static bool
-noise_counter_recv (noise_counter_t * ctr, uint64_t recv)
-{
-  uint64_t i, top, index_recv, index_ctr;
-  unsigned long bit;
-  bool ret = false;
-
-  /* Check that the recv counter is valid */
-  if (ctr->c_recv >= REJECT_AFTER_MESSAGES || recv >= REJECT_AFTER_MESSAGES)
-    goto error;
-
-  /* If the packet is out of the window, invalid */
-  if (recv + COUNTER_WINDOW_SIZE < ctr->c_recv)
-    goto error;
-
-  /* If the new counter is ahead of the current counter, we'll need to
-   * zero out the bitmap that has previously been used */
-  index_recv = recv / COUNTER_BITS;
-  index_ctr = ctr->c_recv / COUNTER_BITS;
-
-  if (recv > ctr->c_recv)
-    {
-      top = clib_min (index_recv - index_ctr, COUNTER_NUM);
-      for (i = 1; i <= top; i++)
-	ctr->c_backtrack[(i + index_ctr) & (COUNTER_NUM - 1)] = 0;
-      ctr->c_recv = recv;
-    }
-
-  index_recv %= COUNTER_NUM;
-  bit = 1ul << (recv % COUNTER_BITS);
-
-  if (ctr->c_backtrack[index_recv] & bit)
-    goto error;
-
-  ctr->c_backtrack[index_recv] |= bit;
-
-  ret = true;
-error:
-  return ret;
-}
-
 static void
 noise_kdf (uint8_t * a, uint8_t * b, uint8_t * c, const uint8_t * x,
 	   size_t a_len, size_t b_len, size_t c_len, size_t x_len,

src/plugins/wireguard/wireguard_noise.h

@@ -136,6 +136,14 @@ noise_local_get (uint32_t locali)
   return (pool_elt_at_index (noise_local_pool, locali));
 }
 
+static_always_inline uint64_t
+noise_counter_send (noise_counter_t *ctr)
+{
+  uint64_t ret;
+  ret = ctr->c_send++;
+  return ret;
+}
+
 void noise_local_init (noise_local_t *, struct noise_upcall *);
 bool noise_local_set_private (noise_local_t *,
 			      const uint8_t[NOISE_PUBLIC_KEY_LEN]);
@@ -187,12 +195,83 @@ noise_remote_encrypt (vlib_main_t * vm, noise_remote_t *,
 		      uint32_t * r_idx,
 		      uint64_t * nonce,
 		      uint8_t * src, size_t srclen, uint8_t * dst);
-enum noise_state_crypt
-noise_remote_decrypt (vlib_main_t * vm, noise_remote_t *,
-		      uint32_t r_idx,
-		      uint64_t nonce,
-		      uint8_t * src, size_t srclen, uint8_t * dst);
 
+static_always_inline noise_keypair_t *
+wg_get_active_keypair (noise_remote_t *r, uint32_t r_idx)
+{
+  if (r->r_current != NULL && r->r_current->kp_local_index == r_idx)
+    {
+      return r->r_current;
+    }
+  else if (r->r_previous != NULL && r->r_previous->kp_local_index == r_idx)
+    {
+      return r->r_previous;
+    }
+  else if (r->r_next != NULL && r->r_next->kp_local_index == r_idx)
+    {
+      return r->r_next;
+    }
+  else
+    {
+      return NULL;
+    }
+}
+
+inline bool
+noise_counter_recv (noise_counter_t *ctr, uint64_t recv)
+{
+  uint64_t i, top, index_recv, index_ctr;
+  unsigned long bit;
+  bool ret = false;
+
+  /* Check that the recv counter is valid */
+  if (ctr->c_recv >= REJECT_AFTER_MESSAGES || recv >= REJECT_AFTER_MESSAGES)
+    goto error;
+
+  /* If the packet is out of the window, invalid */
+  if (recv + COUNTER_WINDOW_SIZE < ctr->c_recv)
+    goto error;
+
+  /* If the new counter is ahead of the current counter, we'll need to
+   * zero out the bitmap that has previously been used */
+  index_recv = recv / COUNTER_BITS;
+  index_ctr = ctr->c_recv / COUNTER_BITS;
+
+  if (recv > ctr->c_recv)
+    {
+      top = clib_min (index_recv - index_ctr, COUNTER_NUM);
+      for (i = 1; i <= top; i++)
+	ctr->c_backtrack[(i + index_ctr) & (COUNTER_NUM - 1)] = 0;
+      ctr->c_recv = recv;
+    }
+
+  index_recv %= COUNTER_NUM;
+  bit = 1ul << (recv % COUNTER_BITS);
+
+  if (ctr->c_backtrack[index_recv] & bit)
+    goto error;
+
+  ctr->c_backtrack[index_recv] |= bit;
+
+  ret = true;
+error:
+  return ret;
+}
+
+static_always_inline void
+noise_remote_keypair_free (vlib_main_t *vm, noise_remote_t *r,
+			   noise_keypair_t **kp)
+{
+  noise_local_t *local = noise_local_get (r->r_local_idx);
+  struct noise_upcall *u = &local->l_upcall;
+  if (*kp)
+    {
+      u->u_index_drop ((*kp)->kp_local_index);
+      vnet_crypto_key_del (vm, (*kp)->kp_send_index);
+      vnet_crypto_key_del (vm, (*kp)->kp_recv_index);
+      clib_mem_free (*kp);
+    }
+}
 
 #endif /* __included_wg_noise_h__ */
 

src/plugins/wireguard/wireguard_timer.c

@@ -26,6 +26,13 @@ get_random_u32_max (u32 max)
   return random_u32 (&seed) % max;
 }
 
+static u32
+get_random_u32_max_opt (u32 max, f64 time)
+{
+  u32 seed = (u32) (time * 1e6);
+  return random_u32 (&seed) % max;
+}
+
 static void
 stop_timer (wg_peer_t * peer, u32 timer_id)
 {
@@ -66,7 +73,7 @@ start_timer_thread_fn (void *arg)
   return 0;
 }
 
-static void
+static_always_inline void
 start_timer_from_mt (u32 peer_idx, u32 timer_id, u32 interval_ticks)
 {
   wg_timers_args a = {
@@ -197,8 +204,8 @@ wg_expired_zero_key_material (vlib_main_t * vm, wg_peer_t * peer)
     }
 }
 
-void
-wg_timers_any_authenticated_packet_traversal (wg_peer_t * peer)
+inline void
+wg_timers_any_authenticated_packet_traversal (wg_peer_t *peer)
 {
   if (peer->persistent_keepalive_interval)
     {
@@ -214,6 +221,12 @@ wg_timers_any_authenticated_packet_sent (wg_peer_t * peer)
   peer->last_sent_packet = vlib_time_now (vlib_get_main ());
 }
 
+inline void
+wg_timers_any_authenticated_packet_sent_opt (wg_peer_t *peer, f64 time)
+{
+  peer->last_sent_packet = time;
+}
+
 void
 wg_timers_handshake_initiated (wg_peer_t * peer)
 {
@@ -246,6 +259,17 @@ wg_timers_data_sent (wg_peer_t * peer)
 		       peer->new_handshake_interval_tick);
 }
 
+inline void
+wg_timers_data_sent_opt (wg_peer_t *peer, f64 time)
+{
+  peer->new_handshake_interval_tick =
+    (KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) * WHZ +
+    get_random_u32_max_opt (REKEY_TIMEOUT_JITTER, time);
+
+  start_timer_from_mt (peer - wg_peer_pool, WG_TIMER_NEW_HANDSHAKE,
+		       peer->new_handshake_interval_tick);
+}
+
 /* Should be called after an authenticated data packet is received. */
 void
 wg_timers_data_received (wg_peer_t * peer)
@@ -275,6 +299,12 @@ wg_timers_any_authenticated_packet_received (wg_peer_t * peer)
   peer->last_received_packet = vlib_time_now (vlib_get_main ());
 }
 
+inline void
+wg_timers_any_authenticated_packet_received_opt (wg_peer_t *peer, f64 time)
+{
+  peer->last_received_packet = time;
+}
+
 static vlib_node_registration_t wg_timer_mngr_node;
 
 static void

src/plugins/wireguard/wireguard_timer.h

@@ -41,9 +41,13 @@ typedef struct wg_peer wg_peer_t;
 void wg_timer_wheel_init ();
 void wg_timers_stop (wg_peer_t * peer);
 void wg_timers_data_sent (wg_peer_t * peer);
+void wg_timers_data_sent_opt (wg_peer_t *peer, f64 time);
 void wg_timers_data_received (wg_peer_t * peer);
 void wg_timers_any_authenticated_packet_sent (wg_peer_t * peer);
+void wg_timers_any_authenticated_packet_sent_opt (wg_peer_t *peer, f64 time);
 void wg_timers_any_authenticated_packet_received (wg_peer_t * peer);
+void wg_timers_any_authenticated_packet_received_opt (wg_peer_t *peer,
+						      f64 time);
 void wg_timers_handshake_initiated (wg_peer_t * peer);
 void wg_timers_handshake_complete (wg_peer_t * peer);
 void wg_timers_session_derived (wg_peer_t * peer);
@@ -57,6 +61,13 @@ wg_birthdate_has_expired (f64 birthday_seconds, f64 expiration_seconds)
   return (birthday_seconds + expiration_seconds) < now_seconds;
 }
 
+static_always_inline bool
+wg_birthdate_has_expired_opt (f64 birthday_seconds, f64 expiration_seconds,
+			      f64 time)
+{
+  return (birthday_seconds + expiration_seconds) < time;
+}
+
 #endif /* __included_wg_timer_h__ */
 
 /*

src/vnet/CMakeLists.txt

@@ -679,6 +679,7 @@ list(APPEND VNET_MULTIARCH_SOURCES
 list(APPEND VNET_HEADERS
   udp/udp_error.def
   udp/udp.h
+  udp/udp_encap.h
   udp/udp_packet.h
   udp/udp_inlines.h
   udp/udp_local.h
@@ -1202,6 +1203,7 @@ list(APPEND VNET_SOURCES
 list(APPEND VNET_HEADERS
   fib/fib.h
   fib/fib_api.h
+  fib/fib_entry_track.h
   fib/ip4_fib.h
   fib/ip4_fib_8.h
   fib/ip4_fib_16.h

src/vnet/crypto/crypto.h

@@ -82,15 +82,16 @@ typedef enum
 /** async crypto **/
 
 /* CRYPTO_ID, PRETTY_NAME, KEY_LENGTH_IN_BYTES, TAG_LEN, AAD_LEN */
-#define foreach_crypto_aead_async_alg \
-  _(AES_128_GCM, "aes-128-gcm-aad8", 16, 16, 8) \
-  _(AES_128_GCM, "aes-128-gcm-aad12", 16, 16, 12) \
-  _(AES_192_GCM, "aes-192-gcm-aad8", 24, 16, 8) \
-  _(AES_192_GCM, "aes-192-gcm-aad12", 24, 16, 12) \
-  _(AES_256_GCM, "aes-256-gcm-aad8", 32, 16, 8) \
-  _(AES_256_GCM, "aes-256-gcm-aad12", 32, 16, 12) \
-  _(CHACHA20_POLY1305, "chacha20-poly1305-aad8", 32, 16, 8) \
-  _(CHACHA20_POLY1305, "chacha20-poly1305-aad12", 32, 16, 12)
+#define foreach_crypto_aead_async_alg                                         \
+  _ (AES_128_GCM, "aes-128-gcm-aad8", 16, 16, 8)                              \
+  _ (AES_128_GCM, "aes-128-gcm-aad12", 16, 16, 12)                            \
+  _ (AES_192_GCM, "aes-192-gcm-aad8", 24, 16, 8)                              \
+  _ (AES_192_GCM, "aes-192-gcm-aad12", 24, 16, 12)                            \
+  _ (AES_256_GCM, "aes-256-gcm-aad8", 32, 16, 8)                              \
+  _ (AES_256_GCM, "aes-256-gcm-aad12", 32, 16, 12)                            \
+  _ (CHACHA20_POLY1305, "chacha20-poly1305-aad8", 32, 16, 8)                  \
+  _ (CHACHA20_POLY1305, "chacha20-poly1305-aad12", 32, 16, 12)                \
+  _ (CHACHA20_POLY1305, "chacha20-poly1305", 32, 16, 0)
 
 /* CRYPTO_ID, INTEG_ID, PRETTY_NAME, KEY_LENGTH_IN_BYTES, DIGEST_LEN */
 #define foreach_crypto_link_async_alg                                         \
@@ -337,7 +338,7 @@ typedef struct
   i16 crypto_start_offset; /* first buffer offset */
   i16 integ_start_offset;
   /* adj total_length for integ, e.g.4 bytes for IPSec ESN */
-  u16 integ_length_adj;
+  i16 integ_length_adj;
   vnet_crypto_op_status_t status : 8;
   u8 flags; /**< share same VNET_CRYPTO_OP_FLAG_* values */
 } vnet_crypto_async_frame_elt_t;
@@ -621,7 +622,7 @@ static_always_inline void
 vnet_crypto_async_add_to_frame (vlib_main_t *vm, vnet_crypto_async_frame_t *f,
 				u32 key_index, u32 crypto_len,
 				i16 integ_len_adj, i16 crypto_start_offset,
-				u16 integ_start_offset, u32 buffer_index,
+				i16 integ_start_offset, u32 buffer_index,
 				u16 next_node, u8 *iv, u8 *tag, u8 *aad,
 				u8 flags)
 {

src/vnet/devices/tap/tap.c

@@ -77,6 +77,14 @@ virtio_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi,
   return 0;
 }
 
+static clib_error_t *
+virtio_eth_set_max_frame_size (vnet_main_t *vnm, vnet_hw_interface_t *hi,
+			       u32 frame_size)
+{
+  /* nothing for now */
+  return 0;
+}
+
 #define TAP_MAX_INSTANCE 1024
 
 static void
@@ -650,6 +658,7 @@ tap_create_if (vlib_main_t * vm, tap_create_if_args_t * args)
       eir.dev_instance = vif->dev_instance;
       eir.address = vif->mac_addr;
       eir.cb.flag_change = virtio_eth_flag_change;
+      eir.cb.set_max_frame_size = virtio_eth_set_max_frame_size;
       vif->hw_if_index = vnet_eth_register_interface (vnm, &eir);
     }
   else

src/vnet/fib/mpls_fib.h

@@ -31,7 +31,7 @@
  * Type exposure is to allow the DP fast/inlined access
  */
 #define MPLS_FIB_KEY_SIZE 21
-#define MPLS_FIB_DB_SIZE (1 << (MPLS_FIB_KEY_SIZE-1))
+#define MPLS_FIB_DB_SIZE (1 << MPLS_FIB_KEY_SIZE)
 
 /**
  * There are no options for controlling the MPLS flow hash,