ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

crypto: encrypt/decrypt queues sw_scheduler

Browse Source 
Type: improvement

Previously multiple sw crypto scheduler queues per core design
caused unaverage frame processing rate for each async op ID –
the lower the op ID is the highly likely they are processed first.

For example, when a RX core is feeding both encryption and
decryption jobs of the same crypto algorithm to the queues at a
high rate, in the mean time the crypto cores have no enough
cycles to process all: the jobs in the decryption queue are less
likely being processed, causing packet drop.

To improve the situation this patch makes every core only owning
a two queues, one for encrypt operations and one for decrypt.
The queue is changed either after checking each core
or after founding a frame to process.
All crypto jobs with different algorithm are pushed to
thoses queues and are treated evenly.

In addition, the crypto async infra now uses unified dequeue handler,
one per engine. Only the active engine will be registered its
dequeue handler in crypto main.

Signed-off-by: DariuszX Kazimierski <dariuszx.kazimierski@intel.com>
Signed-off-by: PiotrX Kleski <piotrx.kleski@intel.com>
Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com>
Signed-off-by: Jakub Wysocki <jakubx.wysocki@intel.com>
Change-Id: I517ee8e31633980de5e0dd4b05e1d5db5dea760e
This commit is contained in:
Jakub Wysocki
2021-11-30 10:53:03 +00:00
committed by Fan Zhang
parent 70e808b1bf
commit 83b2bb8676
8 changed files with 392 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/plugins/crypto_sw_scheduler/crypto_sw_scheduler.h
+16 -2
View File
@@ -21,18 +21,32 @@
#define CRYPTO_SW_SCHEDULER_QUEUE_SIZE 64
#define CRYPTO_SW_SCHEDULER_QUEUE_MASK (CRYPTO_SW_SCHEDULER_QUEUE_SIZE - 1)
STATIC_ASSERT ((0 == (CRYPTO_SW_SCHEDULER_QUEUE_SIZE &
(CRYPTO_SW_SCHEDULER_QUEUE_SIZE - 1))),
"CRYPTO_SW_SCHEDULER_QUEUE_SIZE is not pow2");
typedef enum crypto_sw_scheduler_queue_type_t_
{
CRYPTO_SW_SCHED_QUEUE_TYPE_ENCRYPT = 0,
CRYPTO_SW_SCHED_QUEUE_TYPE_DECRYPT,
CRYPTO_SW_SCHED_QUEUE_N_TYPES
} crypto_sw_scheduler_queue_type_t;
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
u32 head;
u32 tail;
vnet_crypto_async_frame_t *jobs[0];
vnet_crypto_async_frame_t **jobs;
} crypto_sw_scheduler_queue_t;
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
crypto_sw_scheduler_queue_t *queues[VNET_CRYPTO_ASYNC_OP_N_IDS];
crypto_sw_scheduler_queue_t queue[CRYPTO_SW_SCHED_QUEUE_N_TYPES];
u32 last_serve_lcore_id;
u8 last_serve_encrypt;
u8 last_return_queue;
vnet_crypto_op_t *crypto_ops;
vnet_crypto_op_t *integ_ops;
vnet_crypto_op_t *chained_crypto_ops;
src/plugins/crypto_sw_scheduler/main.c
+248 -215
View File
File diff suppressed because it is too large Load Diff
src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c
+14 -10
View File
@@ -515,6 +515,7 @@ cryptodev_register_cop_hdl (vlib_main_t *vm, u32 eidx)
struct rte_cryptodev_sym_capability_idx cap_aead_idx;
u8 *name;
clib_error_t *error = 0;
u32 ref_cnt = 0;
vec_foreach (cet, cmt->per_thread_data)
{
@@ -550,18 +551,18 @@ cryptodev_register_cop_hdl (vlib_main_t *vm, u32 eidx)
vec_validate (cet->cops, VNET_CRYPTO_FRAME_SIZE - 1);
}
/** INDENT-OFF **/
#define _(a, b, c, d, e, f, g) \
cap_aead_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD; \
cap_aead_idx.algo.aead = RTE_CRYPTO_##b##_##c; \
if (cryptodev_check_cap_support (&cap_aead_idx, g, e, f)) \
{ \
vnet_crypto_register_async_handler ( \
vnet_crypto_register_enqueue_handler ( \
vm, eidx, VNET_CRYPTO_OP_##a##_TAG##e##_AAD##f##_ENC, \
cryptodev_enqueue_aead_aad_##f##_enc, cryptodev_frame_dequeue); \
vnet_crypto_register_async_handler ( \
cryptodev_enqueue_aead_aad_##f##_enc); \
vnet_crypto_register_enqueue_handler ( \
vm, eidx, VNET_CRYPTO_OP_##a##_TAG##e##_AAD##f##_DEC, \
cryptodev_enqueue_aead_aad_##f##_dec, cryptodev_frame_dequeue); \
cryptodev_enqueue_aead_aad_##f##_dec); \
ref_cnt++; \
}
foreach_vnet_aead_crypto_conversion
#undef _
@@ -574,16 +575,19 @@ cryptodev_register_cop_hdl (vlib_main_t *vm, u32 eidx)
if (cryptodev_check_cap_support (&cap_cipher_idx, c, -1, -1) && \
cryptodev_check_cap_support (&cap_auth_idx, -1, e, -1)) \
{ \
vnet_crypto_register_async_handler ( \
vnet_crypto_register_enqueue_handler ( \
vm, eidx, VNET_CRYPTO_OP_##a##_##d##_TAG##e##_ENC, \
cryptodev_enqueue_linked_alg_enc, cryptodev_frame_dequeue); \
vnet_crypto_register_async_handler ( \
cryptodev_enqueue_linked_alg_enc); \
vnet_crypto_register_enqueue_handler ( \
vm, eidx, VNET_CRYPTO_OP_##a##_##d##_TAG##e##_DEC, \
cryptodev_enqueue_linked_alg_dec, cryptodev_frame_dequeue); \
cryptodev_enqueue_linked_alg_dec); \
ref_cnt++; \
}
foreach_cryptodev_link_async_alg
#undef _
/** INDENT-ON **/
if (ref_cnt)
vnet_crypto_register_dequeue_handler (vm, eidx, cryptodev_frame_dequeue);
return 0;
src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c
+15 -10
View File
@@ -639,6 +639,7 @@ cryptodev_register_raw_hdl (vlib_main_t *vm, u32 eidx)
struct rte_cryptodev_sym_capability_idx cap_aead_idx;
u32 support_raw_api = 1, max_ctx_size = 0;
clib_error_t *error = 0;
u8 ref_cnt = 0;
vec_foreach (cinst, cmt->cryptodev_inst)
{
@@ -694,18 +695,18 @@ cryptodev_register_raw_hdl (vlib_main_t *vm, u32 eidx)
vec_free (name);
}
/** INDENT-OFF **/
#define _(a, b, c, d, e, f, g) \
cap_aead_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD; \
cap_aead_idx.algo.aead = RTE_CRYPTO_##b##_##c; \
if (cryptodev_check_cap_support (&cap_aead_idx, g, e, f)) \
{ \
vnet_crypto_register_async_handler ( \
vnet_crypto_register_enqueue_handler ( \
vm, eidx, VNET_CRYPTO_OP_##a##_TAG##e##_AAD##f##_ENC, \
cryptodev_raw_enq_aead_aad_##f##_enc, cryptodev_raw_dequeue); \
vnet_crypto_register_async_handler ( \
cryptodev_raw_enq_aead_aad_##f##_enc); \
vnet_crypto_register_enqueue_handler ( \
vm, eidx, VNET_CRYPTO_OP_##a##_TAG##e##_AAD##f##_DEC, \
cryptodev_raw_enq_aead_aad_##f##_dec, cryptodev_raw_dequeue); \
cryptodev_raw_enq_aead_aad_##f##_dec); \
ref_cnt++; \
}
foreach_vnet_aead_crypto_conversion
#undef _
@@ -718,17 +719,21 @@ cryptodev_register_raw_hdl (vlib_main_t *vm, u32 eidx)
if (cryptodev_check_cap_support (&cap_cipher_idx, c, -1, -1) && \
cryptodev_check_cap_support (&cap_auth_idx, -1, e, -1)) \
{ \
vnet_crypto_register_async_handler ( \
vnet_crypto_register_enqueue_handler ( \
vm, eidx, VNET_CRYPTO_OP_##a##_##d##_TAG##e##_ENC, \
cryptodev_raw_enq_linked_alg_enc, cryptodev_raw_dequeue); \
vnet_crypto_register_async_handler ( \
cryptodev_raw_enq_linked_alg_enc); \
vnet_crypto_register_enqueue_handler ( \
vm, eidx, VNET_CRYPTO_OP_##a##_##d##_TAG##e##_DEC, \
cryptodev_raw_enq_linked_alg_dec, cryptodev_raw_dequeue); \
cryptodev_raw_enq_linked_alg_dec); \
ref_cnt++; \
}
foreach_cryptodev_link_async_alg
#undef _
cmt->is_raw_api = 1;
if (ref_cnt)
vnet_crypto_register_dequeue_handler (vm, eidx, cryptodev_raw_dequeue);
cmt->is_raw_api = 1;
return 0;
src/vnet/crypto/cli.c
+1 -1
View File
@@ -67,7 +67,7 @@ format_vnet_crypto_engine_candidates (u8 * s, va_list * args)
{
vec_foreach (e, cm->engines)
{
if (e->enqueue_handlers[id] && e->dequeue_handlers[id])
if (e->enqueue_handlers[id] && e->dequeue_handler)
{
s = format (s, "%U", format_vnet_crypto_engine, e - cm->engines);
if (ei == e - cm->engines)
src/vnet/crypto/crypto.c
+85 -18
View File
@@ -275,10 +275,9 @@ vnet_crypto_register_ops_handlers (vlib_main_t * vm, u32 engine_index,
}
void
vnet_crypto_register_async_handler (vlib_main_t * vm, u32 engine_index,
vnet_crypto_async_op_id_t opt,
vnet_crypto_frame_enqueue_t * enqueue_hdl,
vnet_crypto_frame_dequeue_t * dequeue_hdl)
vnet_crypto_register_enqueue_handler (vlib_main_t *vm, u32 engine_index,
vnet_crypto_async_op_id_t opt,
vnet_crypto_frame_enqueue_t *enqueue_hdl)
{
vnet_crypto_main_t *cm = &crypto_main;
vnet_crypto_engine_t *ae, *e = vec_elt_at_index (cm->engines, engine_index);
@@ -288,17 +287,14 @@ vnet_crypto_register_async_handler (vlib_main_t * vm, u32 engine_index,
vec_validate_aligned (cm->dequeue_handlers, VNET_CRYPTO_ASYNC_OP_N_IDS,
CLIB_CACHE_LINE_BYTES);
/* both enqueue hdl and dequeue hdl should present */
if (!enqueue_hdl && !dequeue_hdl)
if (!enqueue_hdl)
return;
e->enqueue_handlers[opt] = enqueue_hdl;
e->dequeue_handlers[opt] = dequeue_hdl;
if (otd->active_engine_index_async == ~0)
{
otd->active_engine_index_async = engine_index;
cm->enqueue_handlers[opt] = enqueue_hdl;
cm->dequeue_handlers[opt] = dequeue_hdl;
}
ae = vec_elt_at_index (cm->engines, otd->active_engine_index_async);
@@ -306,12 +302,73 @@ vnet_crypto_register_async_handler (vlib_main_t * vm, u32 engine_index,
{
otd->active_engine_index_async = engine_index;
cm->enqueue_handlers[opt] = enqueue_hdl;
cm->dequeue_handlers[opt] = dequeue_hdl;
}
return;
}
static int
engine_index_cmp (void *v1, void *v2)
{
u32 *a1 = v1;
u32 *a2 = v2;
if (*a1 > *a2)
return 1;
if (*a1 < *a2)
return -1;
return 0;
}
static void
vnet_crypto_update_cm_dequeue_handlers (void)
{
vnet_crypto_main_t *cm = &crypto_main;
vnet_crypto_async_op_data_t *otd;
vnet_crypto_engine_t *e;
u32 *active_engines = 0, *ei, last_ei = ~0, i;
vec_reset_length (cm->dequeue_handlers);
for (i = 0; i < VNET_CRYPTO_ASYNC_OP_N_IDS; i++)
{
otd = cm->async_opt_data + i;
e = cm->engines + otd->active_engine_index_async;
if (!e->dequeue_handler)
continue;
vec_add1 (active_engines, otd->active_engine_index_async);
}
vec_sort_with_function (active_engines, engine_index_cmp);
vec_foreach (ei, active_engines)
{
if (ei[0] == last_ei)
continue;
e = cm->engines + ei[0];
vec_add1 (cm->dequeue_handlers, e->dequeue_handler);
last_ei = ei[0];
}
vec_free (active_engines);
}
void
vnet_crypto_register_dequeue_handler (vlib_main_t *vm, u32 engine_index,
vnet_crypto_frame_dequeue_t *deq_fn)
{
vnet_crypto_main_t *cm = &crypto_main;
vnet_crypto_engine_t *e = vec_elt_at_index (cm->engines, engine_index);
if (!deq_fn)
return;
e->dequeue_handler = deq_fn;
return;
}
void
vnet_crypto_register_key_handler (vlib_main_t * vm, u32 engine_index,
vnet_crypto_key_handler_t * key_handler)
@@ -509,11 +566,11 @@ crypto_set_active_async_engine (vnet_crypto_async_op_data_t * od,
vnet_crypto_main_t *cm = &crypto_main;
vnet_crypto_engine_t *ce = vec_elt_at_index (cm->engines, ei);
if (ce->enqueue_handlers[id] && ce->dequeue_handlers[id])
if (ce->enqueue_handlers[id] && ce->dequeue_handler)
{
od->active_engine_index_async = ei;
cm->enqueue_handlers[id] = ce->enqueue_handlers[id];
cm->dequeue_handlers[id] = ce->dequeue_handlers[id];
cm->dequeue_handlers[id] = ce->dequeue_handler;
}
}
@@ -525,6 +582,9 @@ vnet_crypto_set_async_handler2 (char *alg_name, char *engine)
vnet_crypto_async_alg_data_t *ad;
int i;
if (cm->async_refcnt)
return -EBUSY;
p = hash_get_mem (cm->async_alg_index_by_name, alg_name);
if (!p)
return -1;
@@ -546,6 +606,8 @@ vnet_crypto_set_async_handler2 (char *alg_name, char *engine)
crypto_set_active_async_engine (od, id, p[0]);
}
vnet_crypto_update_cm_dequeue_handlers ();
return 0;
}
@@ -603,12 +665,18 @@ vnet_crypto_request_async_mode (int is_enable)
}
if (state_change)
for (i = skip_master; i < tm->n_vlib_mains; i++)
{
vlib_main_t *ovm = vlib_get_main_by_index (i);
if (state != vlib_node_get_state (ovm, cm->crypto_node_index))
vlib_node_set_state (ovm, cm->crypto_node_index, state);
}
{
for (i = skip_master; i < tm->n_vlib_mains; i++)
{
vlib_main_t *ovm = vlib_get_main_by_index (i);
if (state != vlib_node_get_state (ovm, cm->crypto_node_index))
vlib_node_set_state (ovm, cm->crypto_node_index, state);
}
if (is_enable)
vnet_crypto_update_cm_dequeue_handlers ();
}
if (is_enable)
cm->async_refcnt += 1;
@@ -753,7 +821,6 @@ vnet_crypto_init (vlib_main_t * vm)
CLIB_CACHE_LINE_BYTES);
vec_validate (cm->algs, VNET_CRYPTO_N_ALGS);
vec_validate (cm->async_algs, VNET_CRYPTO_N_ASYNC_ALGS);
clib_bitmap_validate (cm->async_active_ids, VNET_CRYPTO_ASYNC_OP_N_IDS);
#define _(n, s, l) \
vnet_crypto_init_cipher_data (VNET_CRYPTO_ALG_##n, \
src/vnet/crypto/crypto.h
+10 -10
View File
@@ -422,12 +422,15 @@ void vnet_crypto_register_key_handler (vlib_main_t * vm, u32 engine_index,
/** async crypto register functions */
u32 vnet_crypto_register_post_node (vlib_main_t * vm, char *post_node_name);
void vnet_crypto_register_async_handler (vlib_main_t * vm,
u32 engine_index,
vnet_crypto_async_op_id_t opt,
vnet_crypto_frame_enqueue_t * enq_fn,
vnet_crypto_frame_dequeue_t *
deq_fn);
void
vnet_crypto_register_enqueue_handler (vlib_main_t *vm, u32 engine_index,
vnet_crypto_async_op_id_t opt,
vnet_crypto_frame_enqueue_t *enq_fn);
void
vnet_crypto_register_dequeue_handler (vlib_main_t *vm, u32 engine_index,
vnet_crypto_frame_dequeue_t *deq_fn);
typedef struct
{
@@ -439,7 +442,7 @@ typedef struct
vnet_crypto_chained_ops_handler_t
* chained_ops_handlers[VNET_CRYPTO_N_OP_IDS];
vnet_crypto_frame_enqueue_t *enqueue_handlers[VNET_CRYPTO_ASYNC_OP_N_IDS];
vnet_crypto_frame_dequeue_t *dequeue_handlers[VNET_CRYPTO_ASYNC_OP_N_IDS];
vnet_crypto_frame_dequeue_t *dequeue_handler;
} vnet_crypto_engine_t;
typedef struct
@@ -456,7 +459,6 @@ typedef struct
vnet_crypto_chained_ops_handler_t **chained_ops_handlers;
vnet_crypto_frame_enqueue_t **enqueue_handlers;
vnet_crypto_frame_dequeue_t **dequeue_handlers;
clib_bitmap_t *async_active_ids;
vnet_crypto_op_data_t opt_data[VNET_CRYPTO_N_OP_IDS];
vnet_crypto_async_op_data_t async_opt_data[VNET_CRYPTO_ASYNC_OP_N_IDS];
vnet_crypto_engine_t *engines;
@@ -591,7 +593,6 @@ vnet_crypto_async_submit_open_frame (vlib_main_t * vm,
{
vnet_crypto_main_t *cm = &crypto_main;
vlib_thread_main_t *tm = vlib_get_thread_main ();
vnet_crypto_async_op_id_t opt = frame->op;
u32 i = vlib_num_workers () > 0;
frame->state = VNET_CRYPTO_FRAME_STATE_PENDING;
@@ -599,7 +600,6 @@ vnet_crypto_async_submit_open_frame (vlib_main_t * vm,
int ret = (cm->enqueue_handlers[frame->op]) (vm, frame);
clib_bitmap_set_no_check (cm->async_active_ids, opt, 1);
if (PREDICT_TRUE (ret == 0))
{
if (cm->dispatch_mode == VNET_CRYPTO_ASYNC_DISPATCH_INTERRUPT)
src/vnet/crypto/node.c
+3 -5
View File
@@ -158,14 +158,12 @@ VLIB_NODE_FN (crypto_dispatch_node) (vlib_main_t * vm,
{
vnet_crypto_main_t *cm = &crypto_main;
vnet_crypto_thread_t *ct = cm->threads + vm->thread_index;
u32 n_dispatched = 0, n_cache = 0;
u32 index;
u32 n_dispatched = 0, n_cache = 0, index;
vec_foreach_index (index, cm->dequeue_handlers)
/* *INDENT-OFF* */
clib_bitmap_foreach (index, cm->async_active_ids) {
n_cache = crypto_dequeue_frame (vm, node, ct, cm->dequeue_handlers[index],
n_cache, &n_dispatched);
}
/* *INDENT-ON* */
if (n_cache)
vlib_buffer_enqueue_to_next_vec (vm, node, &ct->buffer_indices, &ct->nexts,