ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

acl-plugin: make the ACL plugin multicore-capable

Browse Source 
Add the logic to be able to use stateful ACLs in a multithreaded setup.

Change-Id: I3b0cfa6ca4ea8f46f61648611c3e97b00c3376b6
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>
This commit is contained in:
Andrew Yourtchenko
2017-05-17 21:27:03 +02:00
committed by Damjan Marion
parent a62699954a
commit 5dbfbb7110
5 changed files with 914 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/plugins/acl/acl-plugin.md
+347
View File
File diff suppressed because it is too large Load Diff
src/plugins/acl/acl.c
+34 -5
View File
@@ -1785,6 +1785,7 @@ done:
return error;
}
static clib_error_t *
acl_show_aclplugin_fn (vlib_main_t * vm,
unformat_input_t * input,
@@ -1799,6 +1800,7 @@ acl_show_aclplugin_fn (vlib_main_t * vm,
if (unformat (input, "sessions"))
{
u8 * out0 = 0;
u16 wk;
pool_foreach (swif, im->sw_interfaces,
({
u32 sw_if_index = swif->sw_if_index;
@@ -1806,6 +1808,24 @@ acl_show_aclplugin_fn (vlib_main_t * vm,
u64 n_dels = sw_if_index < vec_len(am->fa_session_dels_by_sw_if_index) ? am->fa_session_dels_by_sw_if_index[sw_if_index] : 0;
out0 = format(out0, "sw_if_index %d: add %lu - del %lu = %lu\n", sw_if_index, n_adds, n_dels, n_adds - n_dels);
}));
out0 = format(out0, "\n\nPer-worker data:\n");
for (wk = 0; wk < vec_len (am->per_worker_data); wk++) {
acl_fa_per_worker_data_t *pw = &am->per_worker_data[wk];
out0 = format(out0, "Worker #%d:\n", wk);
out0 = format(out0, " Next expiry time: %lu\n", pw->next_expiry_time);
out0 = format(out0, " Requeue until time: %lu\n", pw->requeue_until_time);
out0 = format(out0, " Current time wait interval: %lu\n", pw->current_time_wait_interval);
out0 = format(out0, " Count of deleted sessions: %lu\n", pw->cnt_deleted_sessions);
out0 = format(out0, " Delete already deleted: %lu\n", pw->cnt_already_deleted_sessions);
out0 = format(out0, " Session timers restarted: %lu\n", pw->cnt_session_timer_restarted);
out0 = format(out0, " Swipe until this time: %lu\n", pw->swipe_end_time);
out0 = format(out0, " sw_if_index serviced bitmap: %U\n", format_bitmap_hex, pw->serviced_sw_if_index_bitmap);
out0 = format(out0, " pending clear intfc bitmap : %U\n", format_bitmap_hex, pw->pending_clear_sw_if_index_bitmap);
out0 = format(out0, " clear in progress: %u\n", pw->clear_in_process);
out0 = format(out0, " interrupt is pending: %d\n", pw->interrupt_is_pending);
out0 = format(out0, " interrupt is needed: %d\n", pw->interrupt_is_needed);
out0 = format(out0, " interrupt is unwanted: %d\n", pw->interrupt_is_unwanted);
}
out0 = format(out0, "\n\nConn cleaner thread counters:\n");
#define _(cnt, desc) out0 = format(out0, " %20lu: %s\n", am->cnt, desc);
foreach_fa_cleaner_counter;
@@ -1867,14 +1887,24 @@ acl_init (vlib_main_t * vm)
am->fa_conn_table_hash_num_buckets = ACL_FA_CONN_TABLE_DEFAULT_HASH_NUM_BUCKETS;
am->fa_conn_table_hash_memory_size = ACL_FA_CONN_TABLE_DEFAULT_HASH_MEMORY_SIZE;
am->fa_conn_table_max_entries = ACL_FA_CONN_TABLE_DEFAULT_MAX_ENTRIES;
vlib_thread_main_t *tm = vlib_get_thread_main ();
// vec_validate(am->per_worker_data, os_get_nthreads()-1);
vec_validate(am->per_worker_data, tm->n_vlib_mains-1);
clib_warning("ACL_FA_INIT: per-worker len: %d", vec_len(am->per_worker_data));
{
u16 wk;
u8 tt;
for(tt = 0; tt < ACL_N_TIMEOUTS; tt++) {
am->fa_conn_list_head[tt] = ~0;
am->fa_conn_list_tail[tt] = ~0;
for (wk = 0; wk < vec_len (am->per_worker_data); wk++) {
acl_fa_per_worker_data_t *pw = &am->per_worker_data[wk];
vec_validate(pw->fa_conn_list_head, ACL_N_TIMEOUTS-1);
vec_validate(pw->fa_conn_list_tail, ACL_N_TIMEOUTS-1);
for(tt = 0; tt < ACL_N_TIMEOUTS; tt++) {
pw->fa_conn_list_head[tt] = ~0;
pw->fa_conn_list_tail[tt] = ~0;
}
}
}
clib_warning("ACL_FA_INIT-DONE: per-worker len: %d", vec_len(am->per_worker_data));
am->fa_min_deleted_sessions_per_interval = ACL_FA_DEFAULT_MIN_DELETED_SESSIONS_PER_INTERVAL;
am->fa_max_deleted_sessions_per_interval = ACL_FA_DEFAULT_MAX_DELETED_SESSIONS_PER_INTERVAL;
@@ -1883,7 +1913,6 @@ acl_init (vlib_main_t * vm)
am->fa_cleaner_cnt_delete_by_sw_index = 0;
am->fa_cleaner_cnt_delete_by_sw_index_ok = 0;
am->fa_cleaner_cnt_unknown_event = 0;
am->fa_cleaner_cnt_deleted_sessions = 0;
am->fa_cleaner_cnt_timer_restarted = 0;
am->fa_cleaner_cnt_wait_with_timeout = 0;
src/plugins/acl/acl.h
+4 -7
View File
@@ -138,9 +138,7 @@ typedef struct {
/* bitmap, when set the hash is initialized */
uword *fa_sessions_on_sw_if_index;
clib_bihash_40_8_t *fa_sessions_by_sw_if_index;
/* pool for FA session data. See fa_node.h */
fa_session_t *fa_sessions_pool;
/* The process node which is responsible to deleting the sessions */
/* The process node which orcherstrates the cleanup */
u32 fa_cleaner_node_index;
/* FA session timeouts, in seconds */
u32 session_timeout_sec[ACL_N_TIMEOUTS];
@@ -192,8 +190,9 @@ typedef struct {
f64 fa_cleaner_wait_time_increment;
u64 fa_current_cleaner_timer_wait_interval;
u32 fa_conn_list_head[ACL_N_TIMEOUTS];
u32 fa_conn_list_tail[ACL_N_TIMEOUTS];
/* per-worker data related t conn management */
acl_fa_per_worker_data_t *per_worker_data;
/* Configured session timeout */
u64 session_timeout[ACL_N_TIMEOUTS];
@@ -205,12 +204,10 @@ typedef struct {
_(fa_cleaner_cnt_delete_by_sw_index, "delete_by_sw_index events") \
_(fa_cleaner_cnt_delete_by_sw_index_ok, "delete_by_sw_index handled ok") \
_(fa_cleaner_cnt_unknown_event, "unknown events received") \
_(fa_cleaner_cnt_deleted_sessions, "sessions deleted") \
_(fa_cleaner_cnt_timer_restarted, "session idle timers restarted") \
_(fa_cleaner_cnt_wait_with_timeout, "event wait with timeout called") \
_(fa_cleaner_cnt_wait_without_timeout, "event wait w/o timeout called") \
_(fa_cleaner_cnt_event_cycles, "total event cycles") \
_(fa_cleaner_cnt_already_deleted, "try to delete already deleted conn") \
/* end of counters */
#define _(id, desc) u32 id;
foreach_fa_cleaner_counter
src/plugins/acl/fa_node.c
+461 -189
View File
File diff suppressed because it is too large Load Diff
src/plugins/acl/fa_node.h
+68 -8
View File
@@ -59,15 +59,29 @@ typedef struct {
u8 as_u8[2];
u16 as_u16;
} tcp_flags_seen; ; /* +2 bytes = 62 */
u8 link_list_id; /* +1 bytes = 63 */
u8 reserved1; /* +1 bytes = 64 */
u32 link_prev_idx;
u32 link_next_idx;
u64 link_enqueue_time;
u64 reserved2[6];
u16 thread_index; /* +2 bytes = 64 */
u64 link_enqueue_time; /* 8 byte = 8 */
u32 link_prev_idx; /* +4 bytes = 12 */
u32 link_next_idx; /* +4 bytes = 16 */
u8 link_list_id; /* +1 bytes = 17 */
u8 reserved1[7]; /* +7 bytes = 24 */
u64 reserved2[5]; /* +5*8 bytes = 64 */
} fa_session_t;
/* This structure is used to fill in the u64 value
in the per-sw-if-index hash table */
typedef struct {
union {
u64 as_u64;
struct {
u32 session_index;
u16 thread_index;
u16 reserved0;
};
};
} fa_full_session_id_t;
/*
* A few compile-time constraints on the size and the layout of the union, to ensure
* it makes sense both for bihash and for us.
@@ -79,10 +93,56 @@ CT_ASSERT_EQUAL(fa_l4_key_t_is_8, sizeof(fa_session_l4_key_t), sizeof(u64));
CT_ASSERT_EQUAL(fa_packet_info_t_is_8, sizeof(fa_packet_info_t), sizeof(u64));
CT_ASSERT_EQUAL(fa_l3_kv_size_is_48, sizeof(fa_5tuple_t), sizeof(clib_bihash_kv_40_8_t));
/* Let's try to fit within the cacheline */
CT_ASSERT_EQUAL(fa_session_t_size_is_64, sizeof(fa_session_t), 128);
/* Let's try to fit within two cachelines */
CT_ASSERT_EQUAL(fa_session_t_size_is_128, sizeof(fa_session_t), 128);
/* Session ID MUST be the same as u64 */
CT_ASSERT_EQUAL(fa_full_session_id_size_is_64, sizeof(fa_full_session_id_t), sizeof(u64));
#undef CT_ASSERT_EQUAL
typedef struct {
/* The pool of sessions managed by this worker */
fa_session_t *fa_sessions_pool;
/* per-worker ACL_N_TIMEOUTS of conn lists */
u32 *fa_conn_list_head;
u32 *fa_conn_list_tail;
/* Vector of expired connections retrieved from lists */
u32 *expired;
/* the earliest next expiry time */
u64 next_expiry_time;
/* if not zero, look at all the elements until their enqueue timestamp is after below one */
u64 requeue_until_time;
/* Current time between the checks */
u64 current_time_wait_interval;
/* Counter of how many sessions we did delete */
u64 cnt_deleted_sessions;
/* Counter of already deleted sessions being deleted - should not increment unless a bug */
u64 cnt_already_deleted_sessions;
/* Number of times we requeued a session to a head of the list */
u64 cnt_session_timer_restarted;
/* swipe up to this enqueue time, rather than following the timeouts */
u64 swipe_end_time;
/* bitmap of sw_if_index serviced by this worker */
uword *serviced_sw_if_index_bitmap;
/* bitmap of sw_if_indices to clear. set by main thread, cleared by worker */
uword *pending_clear_sw_if_index_bitmap;
/* atomic, indicates that the swipe-deletion of connections is in progress */
u32 clear_in_process;
/* Interrupt is pending from main thread */
int interrupt_is_pending;
/*
* Interrupt node on the worker thread sets this if it knows there is
* more work to do, but it has to finish to avoid hogging the
* core for too long.
*/
int interrupt_is_needed;
/*
* Set to indicate that the interrupt node wants to get less interrupts
* because there is not enough work for the current rate.
*/
int interrupt_is_unwanted;
} acl_fa_per_worker_data_t;
typedef enum {
ACL_FA_ERROR_DROP,