crypto-native: add AES-CTR

Type: feature
Change-Id: Iab84226043d8042a99a507767b75e9d4a89cc5c6
Signed-off-by: Damjan Marion <damarion@cisco.com>

src/plugins/crypto_native/CMakeLists.txt

@@ -23,13 +23,13 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*")
   if(compiler_flag_march_alderlake)
     list(APPEND VARIANTS "adl\;-march=alderlake -mprefer-vector-width=256")
   endif()
-  set (COMPILE_FILES aes_cbc.c aes_gcm.c)
+  set (COMPILE_FILES aes_cbc.c aes_gcm.c aes_ctr.c)
   set (COMPILE_OPTS -Wall -fno-common -maes)
 endif()
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)")
   list(APPEND VARIANTS "armv8\;-march=armv8.1-a+crc+crypto")
-  set (COMPILE_FILES aes_cbc.c aes_gcm.c)
+  set (COMPILE_FILES aes_cbc.c aes_gcm.c aes_ctr.c)
   set (COMPILE_OPTS -Wall -fno-common)
 endif()
 

src/plugins/crypto_native/aes_ctr.c

@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright(c) 2024 Cisco Systems, Inc.
+ */
+
+#include <vlib/vlib.h>
+#include <vnet/plugin/plugin.h>
+#include <vnet/crypto/crypto.h>
+#include <crypto_native/crypto_native.h>
+#include <vppinfra/crypto/aes_ctr.h>
+
+#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
+#pragma GCC optimize("O3")
+#endif
+
+static_always_inline u32
+aes_ops_aes_ctr (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops,
+		 vnet_crypto_op_chunk_t *chunks, aes_key_size_t ks,
+		 int maybe_chained)
+{
+  crypto_native_main_t *cm = &crypto_native_main;
+  vnet_crypto_op_t *op = ops[0];
+  aes_ctr_key_data_t *kd;
+  aes_ctr_ctx_t ctx;
+  u32 n_left = n_ops;
+
+next:
+  kd = (aes_ctr_key_data_t *) cm->key_data[op->key_index];
+
+  clib_aes_ctr_init (&ctx, kd, op->iv, ks);
+  if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS)
+    {
+      vnet_crypto_op_chunk_t *chp = chunks + op->chunk_index;
+      for (int j = 0; j < op->n_chunks; j++, chp++)
+	clib_aes_ctr_transform (&ctx, chp->src, chp->dst, chp->len, ks);
+    }
+  else
+    clib_aes_ctr_transform (&ctx, op->src, op->dst, op->len, ks);
+
+  op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
+
+  if (--n_left)
+    {
+      op += 1;
+      goto next;
+    }
+
+  return n_ops;
+}
+
+static_always_inline void *
+aes_ctr_key_exp (vnet_crypto_key_t *key, aes_key_size_t ks)
+{
+  aes_ctr_key_data_t *kd;
+
+  kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES);
+
+  clib_aes_ctr_key_expand (kd, key->data, ks);
+
+  return kd;
+}
+
+#define foreach_aes_ctr_handler_type _ (128) _ (192) _ (256)
+
+#define _(x)                                                                  \
+  static u32 aes_ops_aes_ctr_##x (vlib_main_t *vm, vnet_crypto_op_t *ops[],   \
+				  u32 n_ops)                                  \
+  {                                                                           \
+    return aes_ops_aes_ctr (vm, ops, n_ops, 0, AES_KEY_##x, 0);               \
+  }                                                                           \
+  static u32 aes_ops_aes_ctr_##x##_chained (                                  \
+    vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \
+    u32 n_ops)                                                                \
+  {                                                                           \
+    return aes_ops_aes_ctr (vm, ops, n_ops, chunks, AES_KEY_##x, 1);          \
+  }                                                                           \
+  static void *aes_ctr_key_exp_##x (vnet_crypto_key_t *key)                   \
+  {                                                                           \
+    return aes_ctr_key_exp (key, AES_KEY_##x);                                \
+  }
+
+foreach_aes_ctr_handler_type;
+#undef _
+
+clib_error_t *
+#if defined(__VAES__) && defined(__AVX512F__)
+crypto_native_aes_ctr_init_icl (vlib_main_t *vm)
+#elif defined(__VAES__)
+crypto_native_aes_ctr_init_adl (vlib_main_t *vm)
+#elif __AVX512F__
+crypto_native_aes_ctr_init_skx (vlib_main_t *vm)
+#elif __AVX2__
+crypto_native_aes_ctr_init_hsw (vlib_main_t *vm)
+#elif __aarch64__
+crypto_native_aes_ctr_init_neon (vlib_main_t *vm)
+#else
+crypto_native_aes_ctr_init_slm (vlib_main_t *vm)
+#endif
+{
+  crypto_native_main_t *cm = &crypto_native_main;
+
+#define _(x)                                                                  \
+  vnet_crypto_register_ops_handlers (                                         \
+    vm, cm->crypto_engine_index, VNET_CRYPTO_OP_AES_##x##_CTR_ENC,            \
+    aes_ops_aes_ctr_##x, aes_ops_aes_ctr_##x##_chained);                      \
+  vnet_crypto_register_ops_handlers (                                         \
+    vm, cm->crypto_engine_index, VNET_CRYPTO_OP_AES_##x##_CTR_DEC,            \
+    aes_ops_aes_ctr_##x, aes_ops_aes_ctr_##x##_chained);                      \
+  cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_CTR] = aes_ctr_key_exp_##x;
+  foreach_aes_ctr_handler_type;
+#undef _
+  return 0;
+}

src/plugins/crypto_native/crypto_native.h

@@ -32,9 +32,10 @@ extern crypto_native_main_t crypto_native_main;
 #define foreach_crypto_native_march_variant                                   \
   _ (slm) _ (hsw) _ (skx) _ (icl) _ (adl) _ (neon)
 
-#define _(v) \
+#define _(v)                                                                  \
-clib_error_t __clib_weak *crypto_native_aes_cbc_init_##v (vlib_main_t * vm); \
+  clib_error_t __clib_weak *crypto_native_aes_cbc_init_##v (vlib_main_t *vm); \
-clib_error_t __clib_weak *crypto_native_aes_gcm_init_##v (vlib_main_t * vm); \
+  clib_error_t __clib_weak *crypto_native_aes_ctr_init_##v (vlib_main_t *vm); \
+  clib_error_t __clib_weak *crypto_native_aes_gcm_init_##v (vlib_main_t *vm);
 
 foreach_crypto_native_march_variant;
 #undef _

src/plugins/crypto_native/main.c

@@ -97,6 +97,31 @@ crypto_native_init (vlib_main_t * vm)
   if (error)
     return error;
 
+  if (0)
+    ;
+#if __x86_64__
+  else if (crypto_native_aes_ctr_init_icl && clib_cpu_supports_vaes () &&
+	   clib_cpu_supports_avx512f ())
+    error = crypto_native_aes_ctr_init_icl (vm);
+  else if (crypto_native_aes_ctr_init_adl && clib_cpu_supports_vaes ())
+    error = crypto_native_aes_ctr_init_adl (vm);
+  else if (crypto_native_aes_ctr_init_skx && clib_cpu_supports_avx512f ())
+    error = crypto_native_aes_ctr_init_skx (vm);
+  else if (crypto_native_aes_ctr_init_hsw && clib_cpu_supports_avx2 ())
+    error = crypto_native_aes_ctr_init_hsw (vm);
+  else if (crypto_native_aes_ctr_init_slm)
+    error = crypto_native_aes_ctr_init_slm (vm);
+#endif
+#if __aarch64__
+  else if (crypto_native_aes_ctr_init_neon)
+    error = crypto_native_aes_ctr_init_neon (vm);
+#endif
+  else
+    error = clib_error_return (0, "No AES CTR implemenation available");
+
+  if (error)
+    return error;
+
 #if __x86_64__
   if (clib_cpu_supports_pclmulqdq ())
     {
@@ -133,26 +158,14 @@ crypto_native_init (vlib_main_t * vm)
   return 0;
 }
 
-/* *INDENT-OFF* */
 VLIB_INIT_FUNCTION (crypto_native_init) =
 {
   .runs_after = VLIB_INITS ("vnet_crypto_init"),
 };
-/* *INDENT-ON* */
 
 #include <vpp/app/version.h>
 
-/* *INDENT-OFF* */
 VLIB_PLUGIN_REGISTER () = {
   .version = VPP_BUILD_VER,
-  .description = "Intel IA32 Software Crypto Engine",
+  .description = "Native Crypto Engine",
 };
-/* *INDENT-ON* */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */