wireguard: add async mode for encryption packets

Originally wireguard doesn't support async mode for encryption packets.

This patch add async mode for encryption in wireguard and also adds
support chacha20-poly1305 algorithm in cryptodev for async handler.
In addition it contains new command line to activate async mode for wireguard:
  set wireguard async mode on|off

and also add new command to check active mode for wireguard:
  show wireguard mode

Type: improvement

Signed-off-by: Gabriel Oginski <gabrielx.oginski@intel.com>
Change-Id: I141d48b42ee8dbff0112b8542ab5205268089da6

src/plugins/wireguard/wireguard.api

@@ -195,6 +195,17 @@ define wireguard_peers_details {
   vl_api_wireguard_peer_t peer;
 };
 
+/** \brief Wireguard Set Async mode
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param async_enable - wireguard async mode on or off
+*/
+autoreply define wg_set_async_mode {
+  u32 client_index;
+  u32 context;
+  bool async_enable;
+};
+
 /*
  * Local Variables:
  * eval: (c-set-style "gnu")

src/plugins/wireguard/wireguard.c

@@ -16,6 +16,7 @@
 #include <vnet/vnet.h>
 #include <vnet/plugin/plugin.h>
 #include <vpp/app/version.h>
+#include <vnet/crypto/crypto.h>
 
 #include <wireguard/wireguard_send.h>
 #include <wireguard/wireguard_key.h>
@@ -23,6 +24,31 @@
 #include <wireguard/wireguard.h>
 
 wg_main_t wg_main;
+wg_async_post_next_t wg_encrypt_async_next;
+
+void
+wg_set_async_mode (u32 is_enabled)
+{
+  vnet_crypto_request_async_mode (is_enabled);
+
+  if (is_enabled)
+    wg_op_mode_set_ASYNC ();
+  else
+    wg_op_mode_unset_ASYNC ();
+}
+
+static void
+wireguard_register_post_node (vlib_main_t *vm)
+{
+  wg_async_post_next_t *eit;
+
+  eit = &wg_encrypt_async_next;
+
+  eit->wg4_post_next =
+    vnet_crypto_register_post_node (vm, "wg4-output-tun-post-node");
+  eit->wg6_post_next =
+    vnet_crypto_register_post_node (vm, "wg6-output-tun-post-node");
+}
 
 static clib_error_t *
 wg_init (vlib_main_t * vm)
@@ -44,6 +70,8 @@ wg_init (vlib_main_t * vm)
 			CLIB_CACHE_LINE_BYTES);
 
   wg_timer_wheel_init ();
+  wireguard_register_post_node (vm);
+  wmp->op_mode_flags = 0;
 
   return (NULL);
 }

src/plugins/wireguard/wireguard.h

@@ -30,6 +30,7 @@ typedef struct wg_per_thread_data_t_
 {
   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
   vnet_crypto_op_t *crypto_ops;
+  vnet_crypto_async_frame_t **async_frames;
   u8 data[WG_DEFAULT_DATA_SIZE];
 } wg_per_thread_data_t;
 typedef struct
@@ -50,12 +51,69 @@ typedef struct
   u8 feature_init;
 
   tw_timer_wheel_16t_2w_512sl_t timer_wheel;
+
+  /* operation mode flags (e.g. async) */
+  u8 op_mode_flags;
 } wg_main_t;
 
+typedef struct
+{
+  /* wg post node index for async crypto */
+  u32 wg4_post_next;
+  u32 wg6_post_next;
+} wg_async_post_next_t;
+
+extern wg_async_post_next_t wg_encrypt_async_next;
 extern wg_main_t wg_main;
 
+/**
+ * Wireguard operation mode
+ **/
+#define foreach_wg_op_mode_flags _ (0, ASYNC, "async")
+
+/**
+ * Helper function to set/unset and check op modes
+ **/
+typedef enum wg_op_mode_flags_t_
+{
+#define _(v, f, s) WG_OP_MODE_FLAG_##f = 1 << v,
+  foreach_wg_op_mode_flags
+#undef _
+} __clib_packed wg_op_mode_flags_t;
+
+#define _(a, v, s)                                                            \
+  always_inline int wg_op_mode_set_##v (void)                                 \
+  {                                                                           \
+    return (wg_main.op_mode_flags |= WG_OP_MODE_FLAG_##v);                    \
+  }                                                                           \
+  always_inline int wg_op_mode_unset_##v (void)                               \
+  {                                                                           \
+    return (wg_main.op_mode_flags &= ~WG_OP_MODE_FLAG_##v);                   \
+  }                                                                           \
+  always_inline int wg_op_mode_is_set_##v (void)                              \
+  {                                                                           \
+    return (wg_main.op_mode_flags & WG_OP_MODE_FLAG_##v);                     \
+  }
+foreach_wg_op_mode_flags
+#undef _
+
+  typedef struct
+{
+  u8 __pad[22];
+  u16 next_index;
+} wg_post_data_t;
+
+STATIC_ASSERT (sizeof (wg_post_data_t) <=
+		 STRUCT_SIZE_OF (vnet_buffer_opaque_t, unused),
+	       "Custom meta-data too large for vnet_buffer_opaque_t");
+
+#define wg_post_data(b)                                                       \
+  ((wg_post_data_t *) ((u8 *) ((b)->opaque) +                                 \
+		       STRUCT_OFFSET_OF (vnet_buffer_opaque_t, unused)))
+
 #define WG_START_EVENT	1
 void wg_feature_init (wg_main_t * wmp);
+void wg_set_async_mode (u32 is_enabled);
 
 #endif /* __included_wg_h__ */
 

src/plugins/wireguard/wireguard_api.c

@@ -365,6 +365,18 @@ wg_api_peer_event (index_t peeri, wg_peer_flags flags)
     };
 }
 
+static void
+vl_api_wg_set_async_mode_t_handler (vl_api_wg_set_async_mode_t *mp)
+{
+  wg_main_t *wmp = &wg_main;
+  vl_api_wg_set_async_mode_reply_t *rmp;
+  int rv = 0;
+
+  wg_set_async_mode (mp->async_enable);
+
+  REPLY_MACRO (VL_API_WG_SET_ASYNC_MODE_REPLY);
+}
+
 /* set tup the API message handling tables */
 #include <wireguard/wireguard.api.c>
 static clib_error_t *

src/plugins/wireguard/wireguard_cli.c

@@ -356,6 +356,61 @@ VLIB_CLI_COMMAND (wg_show_itfs_command, static) =
   .short_help = "show wireguard",
   .function = wg_show_if_command_fn,
 };
+
+static clib_error_t *
+wg_set_async_mode_command_fn (vlib_main_t *vm, unformat_input_t *input,
+			      vlib_cli_command_t *cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  int async_enable = 0;
+
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "on"))
+	async_enable = 1;
+      else if (unformat (line_input, "off"))
+	async_enable = 0;
+      else
+	return (clib_error_return (0, "unknown input '%U'",
+				   format_unformat_error, line_input));
+    }
+
+  wg_set_async_mode (async_enable);
+
+  unformat_free (line_input);
+  return (NULL);
+}
+
+VLIB_CLI_COMMAND (wg_set_async_mode_command, static) = {
+  .path = "set wireguard async mode",
+  .short_help = "set wireguard async mode on|off",
+  .function = wg_set_async_mode_command_fn,
+};
+
+static clib_error_t *
+wg_show_mode_command_fn (vlib_main_t *vm, unformat_input_t *input,
+			 vlib_cli_command_t *cmd)
+{
+  vlib_cli_output (vm, "Wireguard mode");
+
+#define _(v, f, s)                                                            \
+  vlib_cli_output (vm, "\t%s: %s", s,                                         \
+		   (wg_op_mode_is_set_##f () ? "enabled" : "disabled"));
+  foreach_wg_op_mode_flags
+#undef _
+
+    return (NULL);
+}
+
+VLIB_CLI_COMMAND (wg_show_modemode_command, static) = {
+  .path = "show wireguard mode",
+  .short_help = "show wireguard mode",
+  .function = wg_show_mode_command_fn,
+};
+
 /* *INDENT-ON* */
 
 /*

src/plugins/wireguard/wireguard_noise.c

@@ -629,58 +629,6 @@ error:
   return ret;
 }
 
-enum noise_state_crypt
-noise_sync_remote_encrypt (vlib_main_t *vm, vnet_crypto_op_t **crypto_ops,
-			   noise_remote_t *r, uint32_t *r_idx, uint64_t *nonce,
-			   uint8_t *src, size_t srclen, uint8_t *dst, u32 bi,
-			   u8 *iv, f64 time)
-{
-  noise_keypair_t *kp;
-  enum noise_state_crypt ret = SC_FAILED;
-
-  if ((kp = r->r_current) == NULL)
-    goto error;
-
-  /* We confirm that our values are within our tolerances. We want:
-   *  - a valid keypair
-   *  - our keypair to be less than REJECT_AFTER_TIME seconds old
-   *  - our receive counter to be less than REJECT_AFTER_MESSAGES
-   *  - our send counter to be less than REJECT_AFTER_MESSAGES
-   */
-  if (!kp->kp_valid ||
-      wg_birthdate_has_expired_opt (kp->kp_birthdate, REJECT_AFTER_TIME,
-				    time) ||
-      kp->kp_ctr.c_recv >= REJECT_AFTER_MESSAGES ||
-      ((*nonce = noise_counter_send (&kp->kp_ctr)) > REJECT_AFTER_MESSAGES))
-    goto error;
-
-  /* We encrypt into the same buffer, so the caller must ensure that buf
-   * has NOISE_AUTHTAG_LEN bytes to store the MAC. The nonce and index
-   * are passed back out to the caller through the provided data pointer. */
-  *r_idx = kp->kp_remote_index;
-
-  wg_prepare_sync_op (vm, crypto_ops, src, srclen, dst, NULL, 0, *nonce,
-		      VNET_CRYPTO_OP_CHACHA20_POLY1305_ENC, kp->kp_send_index,
-		      bi, iv);
-
-  /* If our values are still within tolerances, but we are approaching
-   * the tolerances, we notify the caller with ESTALE that they should
-   * establish a new keypair. The current keypair can continue to be used
-   * until the tolerances are hit. We notify if:
-   *  - our send counter is valid and not less than REKEY_AFTER_MESSAGES
-   *  - we're the initiator and our keypair is older than
-   *    REKEY_AFTER_TIME seconds */
-  ret = SC_KEEP_KEY_FRESH;
-  if ((kp->kp_valid && *nonce >= REKEY_AFTER_MESSAGES) ||
-      (kp->kp_is_initiator && wg_birthdate_has_expired_opt (
-				kp->kp_birthdate, REKEY_AFTER_TIME, time)))
-    goto error;
-
-  ret = SC_OK;
-error:
-  return ret;
-}
-
 enum noise_state_crypt
 noise_sync_remote_decrypt (vlib_main_t *vm, vnet_crypto_op_t **crypto_ops,
 			   noise_remote_t *r, uint32_t r_idx, uint64_t nonce,
@@ -791,14 +739,6 @@ noise_remote_handshake_index_drop (noise_remote_t * r)
     u->u_index_drop (hs->hs_local_index);
 }
 
-static uint64_t
-noise_counter_send (noise_counter_t * ctr)
-{
-  uint64_t ret;
-  ret = ctr->c_send++;
-  return ret;
-}
-
 static void
 noise_kdf (uint8_t * a, uint8_t * b, uint8_t * c, const uint8_t * x,
 	   size_t a_len, size_t b_len, size_t c_len, size_t x_len,

src/plugins/wireguard/wireguard_noise.h

@@ -136,6 +136,14 @@ noise_local_get (uint32_t locali)
   return (pool_elt_at_index (noise_local_pool, locali));
 }
 
+static_always_inline uint64_t
+noise_counter_send (noise_counter_t *ctr)
+{
+  uint64_t ret;
+  ret = ctr->c_send++;
+  return ret;
+}
+
 void noise_local_init (noise_local_t *, struct noise_upcall *);
 bool noise_local_set_private (noise_local_t *,
 			      const uint8_t[NOISE_PUBLIC_KEY_LEN]);
@@ -188,12 +196,6 @@ noise_remote_encrypt (vlib_main_t * vm, noise_remote_t *,
 		      uint64_t * nonce,
 		      uint8_t * src, size_t srclen, uint8_t * dst);
 
-enum noise_state_crypt
-noise_sync_remote_encrypt (vlib_main_t *vm, vnet_crypto_op_t **crypto_ops,
-			   noise_remote_t *r, uint32_t *r_idx, uint64_t *nonce,
-			   uint8_t *src, size_t srclen, uint8_t *dst, u32 bi,
-			   u8 *iv, f64 time);
-
 enum noise_state_crypt
 noise_sync_remote_decrypt (vlib_main_t *vm, vnet_crypto_op_t **crypto_ops,
 			   noise_remote_t *, uint32_t r_idx, uint64_t nonce,

src/plugins/wireguard/wireguard_timer.c

@@ -73,7 +73,7 @@ start_timer_thread_fn (void *arg)
   return 0;
 }
 
-static void
+static_always_inline void
 start_timer_from_mt (u32 peer_idx, u32 timer_id, u32 interval_ticks)
 {
   wg_timers_args a = {
@@ -204,8 +204,8 @@ wg_expired_zero_key_material (vlib_main_t * vm, wg_peer_t * peer)
     }
 }
 
-void
-wg_timers_any_authenticated_packet_traversal (wg_peer_t * peer)
+inline void
+wg_timers_any_authenticated_packet_traversal (wg_peer_t *peer)
 {
   if (peer->persistent_keepalive_interval)
     {
@@ -221,7 +221,7 @@ wg_timers_any_authenticated_packet_sent (wg_peer_t * peer)
   peer->last_sent_packet = vlib_time_now (vlib_get_main ());
 }
 
-void
+inline void
 wg_timers_any_authenticated_packet_sent_opt (wg_peer_t *peer, f64 time)
 {
   peer->last_sent_packet = time;
@@ -259,7 +259,7 @@ wg_timers_data_sent (wg_peer_t * peer)
 		       peer->new_handshake_interval_tick);
 }
 
-void
+inline void
 wg_timers_data_sent_opt (wg_peer_t *peer, f64 time)
 {
   peer->new_handshake_interval_tick =
@@ -299,7 +299,7 @@ wg_timers_any_authenticated_packet_received (wg_peer_t * peer)
   peer->last_received_packet = vlib_time_now (vlib_get_main ());
 }
 
-void
+inline void
 wg_timers_any_authenticated_packet_received_opt (wg_peer_t *peer, f64 time)
 {
   peer->last_received_packet = time;

src/plugins/wireguard/wireguard_timer.h

@@ -61,7 +61,7 @@ wg_birthdate_has_expired (f64 birthday_seconds, f64 expiration_seconds)
   return (birthday_seconds + expiration_seconds) < now_seconds;
 }
 
-static inline bool
+static_always_inline bool
 wg_birthdate_has_expired_opt (f64 birthday_seconds, f64 expiration_seconds,
 			      f64 time)
 {

src/vnet/crypto/crypto.h

@@ -82,15 +82,16 @@ typedef enum
 /** async crypto **/
 
 /* CRYPTO_ID, PRETTY_NAME, KEY_LENGTH_IN_BYTES, TAG_LEN, AAD_LEN */
-#define foreach_crypto_aead_async_alg \
-  _(AES_128_GCM, "aes-128-gcm-aad8", 16, 16, 8) \
-  _(AES_128_GCM, "aes-128-gcm-aad12", 16, 16, 12) \
-  _(AES_192_GCM, "aes-192-gcm-aad8", 24, 16, 8) \
-  _(AES_192_GCM, "aes-192-gcm-aad12", 24, 16, 12) \
-  _(AES_256_GCM, "aes-256-gcm-aad8", 32, 16, 8) \
-  _(AES_256_GCM, "aes-256-gcm-aad12", 32, 16, 12) \
-  _(CHACHA20_POLY1305, "chacha20-poly1305-aad8", 32, 16, 8) \
-  _(CHACHA20_POLY1305, "chacha20-poly1305-aad12", 32, 16, 12)
+#define foreach_crypto_aead_async_alg                                         \
+  _ (AES_128_GCM, "aes-128-gcm-aad8", 16, 16, 8)                              \
+  _ (AES_128_GCM, "aes-128-gcm-aad12", 16, 16, 12)                            \
+  _ (AES_192_GCM, "aes-192-gcm-aad8", 24, 16, 8)                              \
+  _ (AES_192_GCM, "aes-192-gcm-aad12", 24, 16, 12)                            \
+  _ (AES_256_GCM, "aes-256-gcm-aad8", 32, 16, 8)                              \
+  _ (AES_256_GCM, "aes-256-gcm-aad12", 32, 16, 12)                            \
+  _ (CHACHA20_POLY1305, "chacha20-poly1305-aad8", 32, 16, 8)                  \
+  _ (CHACHA20_POLY1305, "chacha20-poly1305-aad12", 32, 16, 12)                \
+  _ (CHACHA20_POLY1305, "chacha20-poly1305", 32, 16, 0)
 
 /* CRYPTO_ID, INTEG_ID, PRETTY_NAME, KEY_LENGTH_IN_BYTES, DIGEST_LEN */
 #define foreach_crypto_link_async_alg                                         \