ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

crypto-native: rename crypto_ia32 to crypto_native

Browse Source 
Type: refactor

Change-Id: I9f21b3bf669ff913ff50afe5459cf52ff987e701
Signed-off-by: Damjan Marion <damarion@cisco.com>
This commit is contained in:
Damjan Marion
2020-01-28 09:55:25 +01:00
committed by Damjan Marion
parent 0d4a61216c
commit 7d08e39a87
9 changed files with 96 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
MAINTAINERS
View File

@@ -186,10 +186,10 @@ I: nhrp
M: Neale Ranns <nranns@cisco.com>
F: src/vnet/nhrp/
Crypto IA32 Plugin
I: crypto-ia32
Crypto native Plugin
I: crypto-native
M: Damjan Marion <damarion@cisco.com>
F: src/plugins/crypto_ia32/
F: src/plugins/crypto_native/
Crypto openssl Plugin
I: crypto-openssl

6
src/plugins/crypto_ia32/CMakeLists.txt → src/plugins/crypto_native/CMakeLists.txt
View File

@@ -15,7 +15,7 @@ if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*")
return()
endif()
add_vpp_plugin(crypto_ia32 SOURCES main.c)
add_vpp_plugin(crypto_native SOURCES main.c)
list(APPEND VARIANTS "sse42\;-march=silvermont")
list(APPEND VARIANTS "avx2\;-march=core-avx2")
@@ -29,9 +29,9 @@ endif()
foreach(VARIANT ${VARIANTS})
list(GET VARIANT 0 v)
list(GET VARIANT 1 f)
set(l crypto_ia32_${v})
set(l crypto_native_${v})
add_library(${l} OBJECT aes_cbc.c aes_gcm.c)
set_target_properties(${l} PROPERTIES POSITION_INDEPENDENT_CODE ON)
target_compile_options(${l} PUBLIC ${f} -Wall -fno-common -maes)
target_sources(crypto_ia32_plugin PRIVATE $<TARGET_OBJECTS:${l}>)
target_sources(crypto_native_plugin PRIVATE $<TARGET_OBJECTS:${l}>)
endforeach()

0
src/plugins/crypto_ia32/FEATURE.yaml → src/plugins/crypto_native/FEATURE.yaml
View File

24
src/plugins/crypto_ia32/aesni.h → src/plugins/crypto_native/aes.h
View File

@@ -20,13 +20,13 @@
typedef enum
{
AESNI_KEY_128 = 0,
AESNI_KEY_192 = 1,
AESNI_KEY_256 = 2,
} aesni_key_size_t;
AES_KEY_128 = 0,
AES_KEY_192 = 1,
AES_KEY_256 = 2,
} aes_key_size_t;
#define AESNI_KEY_ROUNDS(x) (10 + x *2)
#define AESNI_KEY_BYTES(x) (16 + x * 8)
#define AES_KEY_ROUNDS(x) (10 + x * 2)
#define AES_KEY_BYTES(x) (16 + x * 8)
/* AES-NI based AES key expansion based on code samples from
@@ -178,17 +178,17 @@ aes256_key_expand (__m128i * k, u8 * key)
}
static_always_inline void
aes_key_expand (__m128i * k, u8 * key, aesni_key_size_t ks)
aes_key_expand (__m128i * k, u8 * key, aes_key_size_t ks)
{
switch (ks)
{
case AESNI_KEY_128:
case AES_KEY_128:
aes128_key_expand (k, key);
break;
case AESNI_KEY_192:
case AES_KEY_192:
aes192_key_expand (k, key);
break;
case AESNI_KEY_256:
case AES_KEY_256:
aes256_key_expand (k, key);
break;
}
@@ -196,9 +196,9 @@ aes_key_expand (__m128i * k, u8 * key, aesni_key_size_t ks)
static_always_inline void
aes_key_enc_to_dec (__m128i * k, aesni_key_size_t ks)
aes_key_enc_to_dec (__m128i * k, aes_key_size_t ks)
{
int rounds = AESNI_KEY_ROUNDS (ks);
int rounds = AES_KEY_ROUNDS (ks);
__m128i r;
r = k[rounds];

46
src/plugins/crypto_ia32/aes_cbc.c → src/plugins/crypto_native/aes_cbc.c
View File

@@ -19,8 +19,8 @@
#include <vnet/plugin/plugin.h>
#include <vnet/crypto/crypto.h>
#include <x86intrin.h>
#include <crypto_ia32/crypto_ia32.h>
#include <crypto_ia32/aesni.h>
#include <crypto_native/crypto_native.h>
#include <crypto_native/aes.h>
#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
#pragma GCC optimize ("O3")
@@ -87,7 +87,7 @@ aes_block_store_x4 (u8 * dst[], int i, __m512i r)
static_always_inline void __clib_unused
aes_cbc_dec (__m128i * k, u8 * src, u8 * dst, u8 * iv, int count,
aesni_key_size_t rounds)
aes_key_size_t rounds)
{
__m128i r0, r1, r2, r3, c0, c1, c2, c3, f;
int i;
@@ -152,7 +152,7 @@ aes_cbc_dec (__m128i * k, u8 * src, u8 * dst, u8 * iv, int count,
#ifdef __VAES__
static_always_inline void
vaes_cbc_dec (__m512i * k, u8 * src, u8 * dst, u8 * iv, int count,
aesni_key_size_t rounds)
aes_key_size_t rounds)
{
__m512i permute = { 6, 7, 8, 9, 10, 11, 12, 13 };
__m512i r0, r1, r2, r3, c0, c1, c2, c3, f = { };
@@ -236,12 +236,12 @@ vaes_cbc_dec (__m512i * k, u8 * src, u8 * dst, u8 * iv, int count,
static_always_inline u32
aesni_ops_enc_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[],
u32 n_ops, aesni_key_size_t ks)
u32 n_ops, aes_key_size_t ks)
{
crypto_ia32_main_t *cm = &crypto_ia32_main;
crypto_ia32_per_thread_data_t *ptd = vec_elt_at_index (cm->per_thread_data,
vm->thread_index);
int rounds = AESNI_KEY_ROUNDS (ks);
crypto_native_main_t *cm = &crypto_native_main;
crypto_native_per_thread_data_t *ptd =
vec_elt_at_index (cm->per_thread_data, vm->thread_index);
int rounds = AES_KEY_ROUNDS (ks);
u8 dummy[8192];
u32 i, j, count, n_left = n_ops;
u32xN dummy_mask = { };
@@ -372,10 +372,10 @@ more:
static_always_inline u32
aesni_ops_dec_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[],
u32 n_ops, aesni_key_size_t ks)
u32 n_ops, aes_key_size_t ks)
{
crypto_ia32_main_t *cm = &crypto_ia32_main;
int rounds = AESNI_KEY_ROUNDS (ks);
crypto_native_main_t *cm = &crypto_native_main;
int rounds = AES_KEY_ROUNDS (ks);
vnet_crypto_op_t *op = ops[0];
aes_cbc_key_data_t *kd = (aes_cbc_key_data_t *) cm->key_data[op->key_index];
u32 n_left = n_ops;
@@ -401,7 +401,7 @@ decrypt:
}
static_always_inline void *
aesni_cbc_key_exp (vnet_crypto_key_t * key, aesni_key_size_t ks)
aesni_cbc_key_exp (vnet_crypto_key_t * key, aes_key_size_t ks)
{
__m128i e[15], d[15];
aes_cbc_key_data_t *kd;
@@ -409,7 +409,7 @@ aesni_cbc_key_exp (vnet_crypto_key_t * key, aesni_key_size_t ks)
aes_key_expand (e, key->data, ks);
aes_key_expand (d, key->data, ks);
aes_key_enc_to_dec (d, ks);
for (int i = 0; i < AESNI_KEY_ROUNDS (ks) + 1; i++)
for (int i = 0; i < AES_KEY_ROUNDS (ks) + 1; i++)
{
#if __VAES__
kd->decrypt_key[i] = _mm512_broadcast_i64x2 (d[i]);
@@ -426,12 +426,12 @@ aesni_cbc_key_exp (vnet_crypto_key_t * key, aesni_key_size_t ks)
#define _(x) \
static u32 aesni_ops_dec_aes_cbc_##x \
(vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
{ return aesni_ops_dec_aes_cbc (vm, ops, n_ops, AESNI_KEY_##x); } \
{ return aesni_ops_dec_aes_cbc (vm, ops, n_ops, AES_KEY_##x); } \
static u32 aesni_ops_enc_aes_cbc_##x \
(vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
{ return aesni_ops_enc_aes_cbc (vm, ops, n_ops, AESNI_KEY_##x); } \
{ return aesni_ops_enc_aes_cbc (vm, ops, n_ops, AES_KEY_##x); } \
static void * aesni_cbc_key_exp_##x (vnet_crypto_key_t *key) \
{ return aesni_cbc_key_exp (key, AESNI_KEY_##x); }
{ return aesni_cbc_key_exp (key, AES_KEY_##x); }
foreach_aesni_cbc_handler_type;
#undef _
@@ -440,17 +440,17 @@ foreach_aesni_cbc_handler_type;
clib_error_t *
#ifdef __VAES__
crypto_ia32_aesni_cbc_init_vaes (vlib_main_t * vm)
crypto_native_aes_cbc_init_vaes (vlib_main_t * vm)
#elif __AVX512F__
crypto_ia32_aesni_cbc_init_avx512 (vlib_main_t * vm)
crypto_native_aes_cbc_init_avx512 (vlib_main_t * vm)
#elif __AVX2__
crypto_ia32_aesni_cbc_init_avx2 (vlib_main_t * vm)
crypto_native_aes_cbc_init_avx2 (vlib_main_t * vm)
#else
crypto_ia32_aesni_cbc_init_sse42 (vlib_main_t * vm)
crypto_native_aes_cbc_init_sse42 (vlib_main_t * vm)
#endif
{
crypto_ia32_main_t *cm = &crypto_ia32_main;
crypto_ia32_per_thread_data_t *ptd;
crypto_native_main_t *cm = &crypto_native_main;
crypto_native_per_thread_data_t *ptd;
clib_error_t *err = 0;
int fd;

38
src/plugins/crypto_ia32/aes_gcm.c → src/plugins/crypto_native/aes_gcm.c
View File

@@ -19,9 +19,9 @@
#include <vnet/plugin/plugin.h>
#include <vnet/crypto/crypto.h>
#include <x86intrin.h>
#include <crypto_ia32/crypto_ia32.h>
#include <crypto_ia32/aesni.h>
#include <crypto_ia32/ghash.h>
#include <crypto_native/crypto_native.h>
#include <crypto_native/aes.h>
#include <crypto_native/ghash.h>
#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
#pragma GCC optimize ("O3")
@@ -650,9 +650,9 @@ aes_gcm (const u8 * in, u8 * out, const u8 * addt, const u8 * iv, u8 * tag,
static_always_inline u32
aesni_ops_enc_aes_gcm (vlib_main_t * vm, vnet_crypto_op_t * ops[],
u32 n_ops, aesni_key_size_t ks)
u32 n_ops, aes_key_size_t ks)
{
crypto_ia32_main_t *cm = &crypto_ia32_main;
crypto_native_main_t *cm = &crypto_native_main;
vnet_crypto_op_t *op = ops[0];
aes_gcm_key_data_t *kd;
u32 n_left = n_ops;
@@ -661,7 +661,7 @@ aesni_ops_enc_aes_gcm (vlib_main_t * vm, vnet_crypto_op_t * ops[],
next:
kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
aes_gcm (op->src, op->dst, op->aad, op->iv, op->tag, op->len, op->aad_len,
op->tag_len, kd, AESNI_KEY_ROUNDS (ks), /* is_encrypt */ 1);
op->tag_len, kd, AES_KEY_ROUNDS (ks), /* is_encrypt */ 1);
op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
if (--n_left)
@@ -675,9 +675,9 @@ next:
static_always_inline u32
aesni_ops_dec_aes_gcm (vlib_main_t * vm, vnet_crypto_op_t * ops[],
u32 n_ops, aesni_key_size_t ks)
u32 n_ops, aes_key_size_t ks)
{
crypto_ia32_main_t *cm = &crypto_ia32_main;
crypto_native_main_t *cm = &crypto_native_main;
vnet_crypto_op_t *op = ops[0];
aes_gcm_key_data_t *kd;
u32 n_left = n_ops;
@@ -686,7 +686,7 @@ aesni_ops_dec_aes_gcm (vlib_main_t * vm, vnet_crypto_op_t * ops[],
next:
kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
rv = aes_gcm (op->src, op->dst, op->aad, op->iv, op->tag, op->len,
op->aad_len, op->tag_len, kd, AESNI_KEY_ROUNDS (ks),
op->aad_len, op->tag_len, kd, AES_KEY_ROUNDS (ks),
/* is_encrypt */ 0);
if (rv)
@@ -709,7 +709,7 @@ next:
}
static_always_inline void *
aesni_gcm_key_exp (vnet_crypto_key_t * key, aesni_key_size_t ks)
aesni_gcm_key_exp (vnet_crypto_key_t * key, aes_key_size_t ks)
{
aes_gcm_key_data_t *kd;
__m128i H;
@@ -722,7 +722,7 @@ aesni_gcm_key_exp (vnet_crypto_key_t * key, aesni_key_size_t ks)
/* pre-calculate H */
H = kd->Ke[0];
for (i = 1; i < AESNI_KEY_ROUNDS (ks); i += 1)
for (i = 1; i < AES_KEY_ROUNDS (ks); i += 1)
H = _mm_aesenc_si128 (H, kd->Ke[i]);
H = _mm_aesenclast_si128 (H, kd->Ke[i]);
H = aesni_gcm_bswap (H);
@@ -735,28 +735,28 @@ aesni_gcm_key_exp (vnet_crypto_key_t * key, aesni_key_size_t ks)
#define _(x) \
static u32 aesni_ops_dec_aes_gcm_##x \
(vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
{ return aesni_ops_dec_aes_gcm (vm, ops, n_ops, AESNI_KEY_##x); } \
{ return aesni_ops_dec_aes_gcm (vm, ops, n_ops, AES_KEY_##x); } \
static u32 aesni_ops_enc_aes_gcm_##x \
(vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
{ return aesni_ops_enc_aes_gcm (vm, ops, n_ops, AESNI_KEY_##x); } \
{ return aesni_ops_enc_aes_gcm (vm, ops, n_ops, AES_KEY_##x); } \
static void * aesni_gcm_key_exp_##x (vnet_crypto_key_t *key) \
{ return aesni_gcm_key_exp (key, AESNI_KEY_##x); }
{ return aesni_gcm_key_exp (key, AES_KEY_##x); }
foreach_aesni_gcm_handler_type;
#undef _
clib_error_t *
#ifdef __VAES__
crypto_ia32_aesni_gcm_init_vaes (vlib_main_t * vm)
crypto_native_aes_gcm_init_vaes (vlib_main_t * vm)
#elif __AVX512F__
crypto_ia32_aesni_gcm_init_avx512 (vlib_main_t * vm)
crypto_native_aes_gcm_init_avx512 (vlib_main_t * vm)
#elif __AVX2__
crypto_ia32_aesni_gcm_init_avx2 (vlib_main_t * vm)
crypto_native_aes_gcm_init_avx2 (vlib_main_t * vm)
#else
crypto_ia32_aesni_gcm_init_sse42 (vlib_main_t * vm)
crypto_native_aes_gcm_init_sse42 (vlib_main_t * vm)
#endif
{
crypto_ia32_main_t *cm = &crypto_ia32_main;
crypto_native_main_t *cm = &crypto_native_main;
#define _(x) \
vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \

34
src/plugins/crypto_ia32/crypto_ia32.h → src/plugins/crypto_native/crypto_native.h
View File

@@ -15,36 +15,36 @@
*------------------------------------------------------------------
*/
#ifndef __crypto_ia32_h__
#define __crypto_ia32_h__
#ifndef __crypto_native_h__
#define __crypto_native_h__
typedef void *(crypto_ia32_key_fn_t) (vnet_crypto_key_t * key);
typedef void *(crypto_native_key_fn_t) (vnet_crypto_key_t * key);
typedef struct
{
__m128i cbc_iv[4];
} crypto_ia32_per_thread_data_t;
} crypto_native_per_thread_data_t;
typedef struct
{
u32 crypto_engine_index;
crypto_ia32_per_thread_data_t *per_thread_data;
crypto_ia32_key_fn_t *key_fn[VNET_CRYPTO_N_ALGS];
crypto_native_per_thread_data_t *per_thread_data;
crypto_native_key_fn_t *key_fn[VNET_CRYPTO_N_ALGS];
void **key_data;
} crypto_ia32_main_t;
} crypto_native_main_t;
extern crypto_ia32_main_t crypto_ia32_main;
extern crypto_native_main_t crypto_native_main;
clib_error_t *crypto_ia32_aesni_cbc_init_sse42 (vlib_main_t * vm);
clib_error_t *crypto_ia32_aesni_cbc_init_avx2 (vlib_main_t * vm);
clib_error_t *crypto_ia32_aesni_cbc_init_avx512 (vlib_main_t * vm);
clib_error_t *crypto_ia32_aesni_cbc_init_vaes (vlib_main_t * vm);
clib_error_t *crypto_native_aes_cbc_init_sse42 (vlib_main_t * vm);
clib_error_t *crypto_native_aes_cbc_init_avx2 (vlib_main_t * vm);
clib_error_t *crypto_native_aes_cbc_init_avx512 (vlib_main_t * vm);
clib_error_t *crypto_native_aes_cbc_init_vaes (vlib_main_t * vm);
clib_error_t *crypto_ia32_aesni_gcm_init_sse42 (vlib_main_t * vm);
clib_error_t *crypto_ia32_aesni_gcm_init_avx2 (vlib_main_t * vm);
clib_error_t *crypto_ia32_aesni_gcm_init_avx512 (vlib_main_t * vm);
clib_error_t *crypto_ia32_aesni_gcm_init_vaes (vlib_main_t * vm);
#endif /* __crypto_ia32_h__ */
clib_error_t *crypto_native_aes_gcm_init_sse42 (vlib_main_t * vm);
clib_error_t *crypto_native_aes_gcm_init_avx2 (vlib_main_t * vm);
clib_error_t *crypto_native_aes_gcm_init_avx512 (vlib_main_t * vm);
clib_error_t *crypto_native_aes_gcm_init_vaes (vlib_main_t * vm);
#endif /* __crypto_native_h__ */
/*
* fd.io coding-style-patch-verification: ON

0
src/plugins/crypto_ia32/ghash.h → src/plugins/crypto_native/ghash.h
View File

38
src/plugins/crypto_ia32/main.c → src/plugins/crypto_native/main.c
View File

@@ -18,16 +18,16 @@
#include <vlib/vlib.h>
#include <vnet/plugin/plugin.h>
#include <vnet/crypto/crypto.h>
#include <crypto_ia32/crypto_ia32.h>
#include <crypto_native/crypto_native.h>
crypto_ia32_main_t crypto_ia32_main;
crypto_native_main_t crypto_native_main;
static void
crypto_ia32_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop,
vnet_crypto_key_index_t idx)
crypto_native_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop,
vnet_crypto_key_index_t idx)
{
vnet_crypto_key_t *key = vnet_crypto_get_key (idx);
crypto_ia32_main_t *cm = &crypto_ia32_main;
crypto_native_main_t *cm = &crypto_native_main;
if (cm->key_fn[key->alg] == 0)
return;
@@ -56,9 +56,9 @@ crypto_ia32_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop,
}
clib_error_t *
crypto_ia32_init (vlib_main_t * vm)
crypto_native_init (vlib_main_t * vm)
{
crypto_ia32_main_t *cm = &crypto_ia32_main;
crypto_native_main_t *cm = &crypto_native_main;
vlib_thread_main_t *tm = vlib_get_thread_main ();
clib_error_t *error = 0;
@@ -69,17 +69,17 @@ crypto_ia32_init (vlib_main_t * vm)
CLIB_CACHE_LINE_BYTES);
cm->crypto_engine_index =
vnet_crypto_register_engine (vm, "ia32", 100,
"Intel IA32 ISA Optimized Crypto");
vnet_crypto_register_engine (vm, "native", 100,
"Native ISA Optimized Crypto");
if (clib_cpu_supports_vaes ())
error = crypto_ia32_aesni_cbc_init_vaes (vm);
error = crypto_native_aes_cbc_init_vaes (vm);
else if (clib_cpu_supports_avx512f ())
error = crypto_ia32_aesni_cbc_init_avx512 (vm);
error = crypto_native_aes_cbc_init_avx512 (vm);
else if (clib_cpu_supports_avx2 ())
error = crypto_ia32_aesni_cbc_init_avx2 (vm);
error = crypto_native_aes_cbc_init_avx2 (vm);
else
error = crypto_ia32_aesni_cbc_init_sse42 (vm);
error = crypto_native_aes_cbc_init_sse42 (vm);
if (error)
goto error;
@@ -87,20 +87,20 @@ crypto_ia32_init (vlib_main_t * vm)
if (clib_cpu_supports_pclmulqdq ())
{
if (clib_cpu_supports_vaes ())
error = crypto_ia32_aesni_gcm_init_vaes (vm);
error = crypto_native_aes_gcm_init_vaes (vm);
else if (clib_cpu_supports_avx512f ())
error = crypto_ia32_aesni_gcm_init_avx512 (vm);
error = crypto_native_aes_gcm_init_avx512 (vm);
else if (clib_cpu_supports_avx2 ())
error = crypto_ia32_aesni_gcm_init_avx2 (vm);
error = crypto_native_aes_gcm_init_avx2 (vm);
else
error = crypto_ia32_aesni_gcm_init_sse42 (vm);
error = crypto_native_aes_gcm_init_sse42 (vm);
if (error)
goto error;
}
vnet_crypto_register_key_handler (vm, cm->crypto_engine_index,
crypto_ia32_key_handler);
crypto_native_key_handler);
error:
@@ -111,7 +111,7 @@ error:
}
/* *INDENT-OFF* */
VLIB_INIT_FUNCTION (crypto_ia32_init) =
VLIB_INIT_FUNCTION (crypto_native_init) =
{
.runs_after = VLIB_INITS ("vnet_crypto_init"),
};