ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wireguard: add burst mode

Browse Source 
Originally wireguard does packet by packet encryption and decryption.

This patch adds burst mode for encryption and decryption packets. In
addition, it contains some performance improvement such as prefetching
packet header and reducing the number of current time function calls.

Type: improvement

Signed-off-by: Gabriel Oginski <gabrielx.oginski@intel.com>
Change-Id: I04c7daa9b6dc56cd15c789661a64ec642b35aa3f
This commit is contained in:
Gabriel Oginski
2021-11-04 07:23:08 +00:00
committed by Matthew Smith
parent 0e4e5a8222
commit 8ca08496a4
7 changed files with 539 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/plugins/wireguard/wireguard.h
View File

@@ -28,6 +28,8 @@ extern vlib_node_registration_t wg6_output_tun_node;
typedef struct wg_per_thread_data_t_
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
vnet_crypto_op_t *crypto_ops;
u8 data[WG_DEFAULT_DATA_SIZE];
} wg_per_thread_data_t;
typedef struct

326
src/plugins/wireguard/wireguard_input.c
View File

File diff suppressed because it is too large Load Diff

174
src/plugins/wireguard/wireguard_noise.c
View File

@@ -36,7 +36,7 @@ static uint32_t noise_remote_handshake_index_get (noise_remote_t *);
static void noise_remote_handshake_index_drop (noise_remote_t *);
static uint64_t noise_counter_send (noise_counter_t *);
static bool noise_counter_recv (noise_counter_t *, uint64_t);
bool noise_counter_recv (noise_counter_t *, uint64_t);
static void noise_kdf (uint8_t *, uint8_t *, uint8_t *, const uint8_t *,
size_t, size_t, size_t, size_t,
@@ -407,6 +407,8 @@ noise_remote_begin_session (vlib_main_t * vm, noise_remote_t * r)
/* Now we need to add_new_keypair */
clib_rwlock_writer_lock (&r->r_keypair_lock);
/* Activate barrier to synchronization keys between threads */
vlib_worker_thread_barrier_sync (vm);
next = r->r_next;
current = r->r_current;
previous = r->r_previous;
@@ -438,6 +440,7 @@ noise_remote_begin_session (vlib_main_t * vm, noise_remote_t * r)
r->r_next = noise_remote_keypair_allocate (r);
*r->r_next = kp;
}
vlib_worker_thread_barrier_release (vm);
clib_rwlock_writer_unlock (&r->r_keypair_lock);
secure_zero_memory (&r->r_handshake, sizeof (r->r_handshake));
@@ -541,6 +544,41 @@ chacha20poly1305_calc (vlib_main_t * vm,
return (op->status == VNET_CRYPTO_OP_STATUS_COMPLETED);
}
always_inline void
wg_prepare_sync_op (vlib_main_t *vm, vnet_crypto_op_t **crypto_ops, u8 *src,
u32 src_len, u8 *dst, u8 *aad, u32 aad_len, u64 nonce,
vnet_crypto_op_id_t op_id,
vnet_crypto_key_index_t key_index, u32 bi, u8 *iv)
{
vnet_crypto_op_t _op, *op = &_op;
u8 src_[] = {};
clib_memset (iv, 0, 4);
clib_memcpy (iv + 4, &nonce, sizeof (nonce));
vec_add2_aligned (crypto_ops[0], op, 1, CLIB_CACHE_LINE_BYTES);
vnet_crypto_op_init (op, op_id);
op->tag_len = NOISE_AUTHTAG_LEN;
if (op_id == VNET_CRYPTO_OP_CHACHA20_POLY1305_DEC)
{
op->tag = src + src_len;
op->flags |= VNET_CRYPTO_OP_FLAG_HMAC_CHECK;
}
else
op->tag = dst + src_len;
op->src = !src ? src_ : src;
op->len = src_len;
op->dst = dst;
op->key_index = key_index;
op->aad = aad;
op->aad_len = aad_len;
op->iv = iv;
op->user_data = bi;
}
enum noise_state_crypt
noise_remote_encrypt (vlib_main_t * vm, noise_remote_t * r, uint32_t * r_idx,
uint64_t * nonce, uint8_t * src, size_t srclen,
@@ -592,26 +630,67 @@ error:
}
enum noise_state_crypt
noise_remote_decrypt (vlib_main_t * vm, noise_remote_t * r, uint32_t r_idx,
uint64_t nonce, uint8_t * src, size_t srclen,
uint8_t * dst)
noise_sync_remote_encrypt (vlib_main_t *vm, vnet_crypto_op_t **crypto_ops,
noise_remote_t *r, uint32_t *r_idx, uint64_t *nonce,
uint8_t *src, size_t srclen, uint8_t *dst, u32 bi,
u8 *iv, f64 time)
{
noise_keypair_t *kp;
enum noise_state_crypt ret = SC_FAILED;
if (r->r_current != NULL && r->r_current->kp_local_index == r_idx)
{
kp = r->r_current;
}
else if (r->r_previous != NULL && r->r_previous->kp_local_index == r_idx)
{
kp = r->r_previous;
}
else if (r->r_next != NULL && r->r_next->kp_local_index == r_idx)
{
kp = r->r_next;
}
else
if ((kp = r->r_current) == NULL)
goto error;
/* We confirm that our values are within our tolerances. We want:
* - a valid keypair
* - our keypair to be less than REJECT_AFTER_TIME seconds old
* - our receive counter to be less than REJECT_AFTER_MESSAGES
* - our send counter to be less than REJECT_AFTER_MESSAGES
*/
if (!kp->kp_valid ||
wg_birthdate_has_expired_opt (kp->kp_birthdate, REJECT_AFTER_TIME,
time) ||
kp->kp_ctr.c_recv >= REJECT_AFTER_MESSAGES ||
((*nonce = noise_counter_send (&kp->kp_ctr)) > REJECT_AFTER_MESSAGES))
goto error;
/* We encrypt into the same buffer, so the caller must ensure that buf
* has NOISE_AUTHTAG_LEN bytes to store the MAC. The nonce and index
* are passed back out to the caller through the provided data pointer. */
*r_idx = kp->kp_remote_index;
wg_prepare_sync_op (vm, crypto_ops, src, srclen, dst, NULL, 0, *nonce,
VNET_CRYPTO_OP_CHACHA20_POLY1305_ENC, kp->kp_send_index,
bi, iv);
/* If our values are still within tolerances, but we are approaching
* the tolerances, we notify the caller with ESTALE that they should
* establish a new keypair. The current keypair can continue to be used
* until the tolerances are hit. We notify if:
* - our send counter is valid and not less than REKEY_AFTER_MESSAGES
* - we're the initiator and our keypair is older than
* REKEY_AFTER_TIME seconds */
ret = SC_KEEP_KEY_FRESH;
if ((kp->kp_valid && *nonce >= REKEY_AFTER_MESSAGES) ||
(kp->kp_is_initiator && wg_birthdate_has_expired_opt (
kp->kp_birthdate, REKEY_AFTER_TIME, time)))
goto error;
ret = SC_OK;
error:
return ret;
}
enum noise_state_crypt
noise_sync_remote_decrypt (vlib_main_t *vm, vnet_crypto_op_t **crypto_ops,
noise_remote_t *r, uint32_t r_idx, uint64_t nonce,
uint8_t *src, size_t srclen, uint8_t *dst, u32 bi,
u8 *iv, f64 time)
{
noise_keypair_t *kp;
enum noise_state_crypt ret = SC_FAILED;
if ((kp = wg_get_active_keypair (r, r_idx)) == NULL)
{
goto error;
}
@@ -620,20 +699,17 @@ noise_remote_decrypt (vlib_main_t * vm, noise_remote_t * r, uint32_t r_idx,
* are the same as the encrypt routine.
*
* kp_ctr isn't locked here, we're happy to accept a racy read. */
if (wg_birthdate_has_expired (kp->kp_birthdate, REJECT_AFTER_TIME) ||
if (wg_birthdate_has_expired_opt (kp->kp_birthdate, REJECT_AFTER_TIME,
time) ||
kp->kp_ctr.c_recv >= REJECT_AFTER_MESSAGES)
goto error;
/* Decrypt, then validate the counter. We don't want to validate the
* counter before decrypting as we do not know the message is authentic
* prior to decryption. */
if (!chacha20poly1305_calc (vm, src, srclen, dst, NULL, 0, nonce,
VNET_CRYPTO_OP_CHACHA20_POLY1305_DEC,
kp->kp_recv_index))
goto error;
if (!noise_counter_recv (&kp->kp_ctr, nonce))
goto error;
wg_prepare_sync_op (vm, crypto_ops, src, srclen, dst, NULL, 0, nonce,
VNET_CRYPTO_OP_CHACHA20_POLY1305_DEC, kp->kp_recv_index,
bi, iv);
/* If we've received the handshake confirming data packet then move the
* next keypair into current. If we do slide the next keypair in, then
@@ -662,10 +738,9 @@ noise_remote_decrypt (vlib_main_t * vm, noise_remote_t * r, uint32_t r_idx,
* REKEY_AFTER_TIME_RECV seconds. */
ret = SC_KEEP_KEY_FRESH;
kp = r->r_current;
if (kp != NULL &&
kp->kp_valid &&
kp->kp_is_initiator &&
wg_birthdate_has_expired (kp->kp_birthdate, REKEY_AFTER_TIME_RECV))
if (kp != NULL && kp->kp_valid && kp->kp_is_initiator &&
wg_birthdate_has_expired_opt (kp->kp_birthdate, REKEY_AFTER_TIME_RECV,
time))
goto error;
ret = SC_OK;
@@ -724,47 +799,6 @@ noise_counter_send (noise_counter_t * ctr)
return ret;
}
static bool
noise_counter_recv (noise_counter_t * ctr, uint64_t recv)
{
uint64_t i, top, index_recv, index_ctr;
unsigned long bit;
bool ret = false;
/* Check that the recv counter is valid */
if (ctr->c_recv >= REJECT_AFTER_MESSAGES || recv >= REJECT_AFTER_MESSAGES)
goto error;
/* If the packet is out of the window, invalid */
if (recv + COUNTER_WINDOW_SIZE < ctr->c_recv)
goto error;
/* If the new counter is ahead of the current counter, we'll need to
* zero out the bitmap that has previously been used */
index_recv = recv / COUNTER_BITS;
index_ctr = ctr->c_recv / COUNTER_BITS;
if (recv > ctr->c_recv)
{
top = clib_min (index_recv - index_ctr, COUNTER_NUM);
for (i = 1; i <= top; i++)
ctr->c_backtrack[(i + index_ctr) & (COUNTER_NUM - 1)] = 0;
ctr->c_recv = recv;
}
index_recv %= COUNTER_NUM;
bit = 1ul << (recv % COUNTER_BITS);
if (ctr->c_backtrack[index_recv] & bit)
goto error;
ctr->c_backtrack[index_recv] |= bit;
ret = true;
error:
return ret;
}
static void
noise_kdf (uint8_t * a, uint8_t * b, uint8_t * c, const uint8_t * x,
size_t a_len, size_t b_len, size_t c_len, size_t x_len,

78
src/plugins/wireguard/wireguard_noise.h
View File

@@ -187,12 +187,80 @@ noise_remote_encrypt (vlib_main_t * vm, noise_remote_t *,
uint32_t * r_idx,
uint64_t * nonce,
uint8_t * src, size_t srclen, uint8_t * dst);
enum noise_state_crypt
noise_remote_decrypt (vlib_main_t * vm, noise_remote_t *,
uint32_t r_idx,
uint64_t nonce,
uint8_t * src, size_t srclen, uint8_t * dst);
enum noise_state_crypt
noise_sync_remote_encrypt (vlib_main_t *vm, vnet_crypto_op_t **crypto_ops,
noise_remote_t *r, uint32_t *r_idx, uint64_t *nonce,
uint8_t *src, size_t srclen, uint8_t *dst, u32 bi,
u8 *iv, f64 time);
enum noise_state_crypt
noise_sync_remote_decrypt (vlib_main_t *vm, vnet_crypto_op_t **crypto_ops,
noise_remote_t *, uint32_t r_idx, uint64_t nonce,
uint8_t *src, size_t srclen, uint8_t *dst, u32 bi,
u8 *iv, f64 time);
static_always_inline noise_keypair_t *
wg_get_active_keypair (noise_remote_t *r, uint32_t r_idx)
{
if (r->r_current != NULL && r->r_current->kp_local_index == r_idx)
{
return r->r_current;
}
else if (r->r_previous != NULL && r->r_previous->kp_local_index == r_idx)
{
return r->r_previous;
}
else if (r->r_next != NULL && r->r_next->kp_local_index == r_idx)
{
return r->r_next;
}
else
{
return NULL;
}
}
inline bool
noise_counter_recv (noise_counter_t *ctr, uint64_t recv)
{
uint64_t i, top, index_recv, index_ctr;
unsigned long bit;
bool ret = false;
/* Check that the recv counter is valid */
if (ctr->c_recv >= REJECT_AFTER_MESSAGES || recv >= REJECT_AFTER_MESSAGES)
goto error;
/* If the packet is out of the window, invalid */
if (recv + COUNTER_WINDOW_SIZE < ctr->c_recv)
goto error;
/* If the new counter is ahead of the current counter, we'll need to
* zero out the bitmap that has previously been used */
index_recv = recv / COUNTER_BITS;
index_ctr = ctr->c_recv / COUNTER_BITS;
if (recv > ctr->c_recv)
{
top = clib_min (index_recv - index_ctr, COUNTER_NUM);
for (i = 1; i <= top; i++)
ctr->c_backtrack[(i + index_ctr) & (COUNTER_NUM - 1)] = 0;
ctr->c_recv = recv;
}
index_recv %= COUNTER_NUM;
bit = 1ul << (recv % COUNTER_BITS);
if (ctr->c_backtrack[index_recv] & bit)
goto error;
ctr->c_backtrack[index_recv] |= bit;
ret = true;
error:
return ret;
}
#endif /* __included_wg_noise_h__ */

100
src/plugins/wireguard/wireguard_output_tun.c
View File

@@ -93,32 +93,67 @@ format_wg_output_tun_trace (u8 * s, va_list * args)
return s;
}
static_always_inline void
wg_output_process_ops (vlib_main_t *vm, vlib_node_runtime_t *node,
vnet_crypto_op_t *ops, vlib_buffer_t *b[], u16 *nexts,
u16 drop_next)
{
u32 n_fail, n_ops = vec_len (ops);
vnet_crypto_op_t *op = ops;
if (n_ops == 0)
return;
n_fail = n_ops - vnet_crypto_process_ops (vm, op, n_ops);
while (n_fail)
{
ASSERT (op - ops < n_ops);
if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
{
u32 bi = op->user_data;
b[bi]->error = node->errors[WG_OUTPUT_ERROR_KEYPAIR];
nexts[bi] = drop_next;
n_fail--;
}
op++;
}
}
/* is_ip4 - inner header flag */
always_inline uword
wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
vlib_frame_t *frame, u8 is_ip4)
{
u32 n_left_from;
u32 *from;
wg_main_t *wmp = &wg_main;
wg_per_thread_data_t *ptd =
vec_elt_at_index (wmp->per_thread_data, vm->thread_index);
u32 *from = vlib_frame_vector_args (frame);
u32 n_left_from = frame->n_vectors;
ip4_udp_wg_header_t *hdr4_out = NULL;
ip6_udp_wg_header_t *hdr6_out = NULL;
message_data_t *message_data_wg = NULL;
vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
u16 nexts[VLIB_FRAME_SIZE], *next;
vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
vnet_crypto_op_t **crypto_ops = &ptd->crypto_ops;
u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
vlib_buffer_t *sync_bufs[VLIB_FRAME_SIZE];
u32 thread_index = vm->thread_index;
from = vlib_frame_vector_args (frame);
n_left_from = frame->n_vectors;
b = bufs;
next = nexts;
u16 n_sync = 0;
u16 drop_next = WG_OUTPUT_NEXT_ERROR;
vlib_get_buffers (vm, from, bufs, n_left_from);
vec_reset_length (ptd->crypto_ops);
wg_peer_t *peer = NULL;
u32 adj_index = 0;
u32 last_adj_index = ~0;
index_t peeri = INDEX_INVALID;
f64 time = clib_time_now (&vm->clib_time) + vm->time_offset;
while (n_left_from > 0)
{
index_t peeri;
u8 iph_offset = 0;
u8 is_ip4_out = 1;
u8 *plain_data;
@@ -130,14 +165,21 @@ wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
vlib_prefetch_buffer_header (b[2], LOAD);
p = vlib_buffer_get_current (b[1]);
CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
CLIB_PREFETCH (vlib_buffer_get_tail (b[1]), CLIB_CACHE_LINE_BYTES,
LOAD);
}
next[0] = WG_OUTPUT_NEXT_ERROR;
peeri =
wg_peer_get_by_adj_index (vnet_buffer (b[0])->ip.adj_index[VLIB_TX]);
peer = wg_peer_get (peeri);
if (wg_peer_is_dead (peer))
adj_index = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
if (PREDICT_FALSE (last_adj_index != adj_index))
{
peeri = wg_peer_get_by_adj_index (adj_index);
peer = wg_peer_get (peeri);
}
if (!peer || wg_peer_is_dead (peer))
{
b[0]->error = node->errors[WG_OUTPUT_ERROR_PEER];
goto out;
@@ -179,6 +221,7 @@ wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
iph_offset = vnet_buffer (b[0])->ip.save_rewrite_length;
plain_data = vlib_buffer_get_current (b[0]) + iph_offset;
plain_data_len = vlib_buffer_length_in_chain (vm, b[0]) - iph_offset;
u8 *iv_data = b[0]->pre_data;
size_t encrypted_packet_len = message_data_len (plain_data_len);
@@ -194,11 +237,20 @@ wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
goto out;
}
if (PREDICT_FALSE (last_adj_index != adj_index))
{
wg_timers_any_authenticated_packet_sent_opt (peer, time);
wg_timers_data_sent_opt (peer, time);
wg_timers_any_authenticated_packet_traversal (peer);
last_adj_index = adj_index;
}
enum noise_state_crypt state;
state = noise_remote_encrypt (
vm, &peer->remote, &message_data_wg->receiver_index,
&message_data_wg->counter, plain_data, plain_data_len, plain_data);
state = noise_sync_remote_encrypt (
vm, crypto_ops, &peer->remote, &message_data_wg->receiver_index,
&message_data_wg->counter, plain_data, plain_data_len, plain_data,
n_sync, iv_data, time);
if (PREDICT_FALSE (state == SC_KEEP_KEY_FRESH))
{
@@ -206,7 +258,7 @@ wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
}
else if (PREDICT_FALSE (state == SC_FAILED))
{
//TODO: Maybe wrong
// TODO: Maybe wrong
wg_send_handshake_from_mt (peeri, false);
wg_peer_update_flags (peeri, WG_PEER_ESTABLISHED, false);
goto out;
@@ -236,10 +288,6 @@ wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
clib_host_to_net_u16 (b[0]->current_length);
}
wg_timers_any_authenticated_packet_sent (peer);
wg_timers_data_sent (peer);
wg_timers_any_authenticated_packet_traversal (peer);
out:
if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
&& (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
@@ -256,12 +304,18 @@ wg_output_tun_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
}
next:
sync_bufs[n_sync] = b[0];
n_sync += 1;
n_left_from -= 1;
next += 1;
b += 1;
}
vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
/* wg-output-process-ops */
wg_output_process_ops (vm, node, ptd->crypto_ops, sync_bufs, nexts,
drop_next);
vlib_buffer_enqueue_to_next (vm, node, from, nexts, n_sync);
return frame->n_vectors;
}

30
src/plugins/wireguard/wireguard_timer.c
View File

@@ -26,6 +26,13 @@ get_random_u32_max (u32 max)
return random_u32 (&seed) % max;
}
static u32
get_random_u32_max_opt (u32 max, f64 time)
{
u32 seed = (u32) (time * 1e6);
return random_u32 (&seed) % max;
}
static void
stop_timer (wg_peer_t * peer, u32 timer_id)
{
@@ -214,6 +221,12 @@ wg_timers_any_authenticated_packet_sent (wg_peer_t * peer)
peer->last_sent_packet = vlib_time_now (vlib_get_main ());
}
void
wg_timers_any_authenticated_packet_sent_opt (wg_peer_t *peer, f64 time)
{
peer->last_sent_packet = time;
}
void
wg_timers_handshake_initiated (wg_peer_t * peer)
{
@@ -246,6 +259,17 @@ wg_timers_data_sent (wg_peer_t * peer)
peer->new_handshake_interval_tick);
}
void
wg_timers_data_sent_opt (wg_peer_t *peer, f64 time)
{
peer->new_handshake_interval_tick =
(KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) * WHZ +
get_random_u32_max_opt (REKEY_TIMEOUT_JITTER, time);
start_timer_from_mt (peer - wg_peer_pool, WG_TIMER_NEW_HANDSHAKE,
peer->new_handshake_interval_tick);
}
/* Should be called after an authenticated data packet is received. */
void
wg_timers_data_received (wg_peer_t * peer)
@@ -275,6 +299,12 @@ wg_timers_any_authenticated_packet_received (wg_peer_t * peer)
peer->last_received_packet = vlib_time_now (vlib_get_main ());
}
void
wg_timers_any_authenticated_packet_received_opt (wg_peer_t *peer, f64 time)
{
peer->last_received_packet = time;
}
static vlib_node_registration_t wg_timer_mngr_node;
static void

11
src/plugins/wireguard/wireguard_timer.h
View File

@@ -41,9 +41,13 @@ typedef struct wg_peer wg_peer_t;
void wg_timer_wheel_init ();
void wg_timers_stop (wg_peer_t * peer);
void wg_timers_data_sent (wg_peer_t * peer);
void wg_timers_data_sent_opt (wg_peer_t *peer, f64 time);
void wg_timers_data_received (wg_peer_t * peer);
void wg_timers_any_authenticated_packet_sent (wg_peer_t * peer);
void wg_timers_any_authenticated_packet_sent_opt (wg_peer_t *peer, f64 time);
void wg_timers_any_authenticated_packet_received (wg_peer_t * peer);
void wg_timers_any_authenticated_packet_received_opt (wg_peer_t *peer,
f64 time);
void wg_timers_handshake_initiated (wg_peer_t * peer);
void wg_timers_handshake_complete (wg_peer_t * peer);
void wg_timers_session_derived (wg_peer_t * peer);
@@ -57,6 +61,13 @@ wg_birthdate_has_expired (f64 birthday_seconds, f64 expiration_seconds)
return (birthday_seconds + expiration_seconds) < now_seconds;
}
static inline bool
wg_birthdate_has_expired_opt (f64 birthday_seconds, f64 expiration_seconds,
f64 time)
{
return (birthday_seconds + expiration_seconds) < time;
}
#endif /* __included_wg_timer_h__ */
/*