ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vlib: refactor checksum offload support

Browse Source 
Type: refactor

This patch refactors the offload flags in vlib_buffer_t.
There are two main reasons behind this refactoring.
First, offload flags are insufficient to represent outer
and inner headers offloads. Second, room for these flags
in first cacheline of vlib_buffer_t is also limited.

This patch introduces a generic offload flag in first
cacheline. And detailed offload flags in 2nd cacheline
of the structure for performance optimization.

Change-Id: Icc363a142fb9208ec7113ab5bbfc8230181f6004
Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>
This commit is contained in:
Mohsin Kazmi 2021-02-10 11:26:24 +01:00 committed by Damjan Marion
parent 99c6dc6a7a
commit 6809538e64
34 changed files with 376 additions and 263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
extras/pg/checksum_offload.pg
View File

@ -15,7 +15,8 @@ packet-generator new {
interface loop0
tx-interface loop1
node loop1-output
buffer-flags ip4 offload-ip-cksum offload-udp-cksum
buffer-flags ip4 offload
buffer-offload-flags offload-ip-cksum offload-udp-cksum
data { IP4: 1.2.3 -> 4.5.6
UDP: 11.22.33.44 -> 11.22.34.44
ttl 2 checksum 13

14
src/plugins/avf/output.c
View File

@ -57,12 +57,12 @@ static_always_inline u64
avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso)
{
u64 flags = 0;
if (!is_tso && !(b->flags & ((VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
if (!is_tso && !(b->flags & VNET_BUFFER_F_OFFLOAD))
return 0;
u32 is_tcp = is_tso || b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
u32 is_udp = !is_tso && b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
u32 oflags = vnet_buffer2 (b)->oflags;
u32 is_tcp = is_tso || oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
u32 is_udp = !is_tso && oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
u32 is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
u32 is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
ASSERT (!is_tcp || !is_udp);
@ -173,9 +173,7 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
{
u16 next = txq->next;
u64 bits = AVF_TXD_CMD_EOP | AVF_TXD_CMD_RSV;
const u32 offload_mask = VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
VNET_BUFFER_F_GSO;
const u32 offload_mask = VNET_BUFFER_F_OFFLOAD | VNET_BUFFER_F_GSO;
u64 one_by_one_offload_flags = 0;
int is_tso;
u16 n_desc = 0;

21
src/plugins/dpdk/device/device.c
View File

@ -218,17 +218,20 @@ static_always_inline void
dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b,
struct rte_mbuf *mb)
{
u32 ip_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
u32 tcp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
u32 udp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
u32 tso = b->flags & VNET_BUFFER_F_GSO, max_pkt_len;
u32 oflags, ip_cksum, tcp_cksum, udp_cksum;
u64 ol_flags;
/* Is there any work for us? */
if (PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum | tso) == 0))
if (PREDICT_TRUE (((b->flags & VNET_BUFFER_F_OFFLOAD) | tso) == 0))
return;
oflags = vnet_buffer2 (b)->oflags;
ip_cksum = oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
tcp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
udp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
mb->l2_len = vnet_buffer (b)->l3_hdr_offset - b->current_data;
mb->l3_len = vnet_buffer (b)->l4_hdr_offset -
vnet_buffer (b)->l3_hdr_offset;
@ -328,10 +331,7 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm,
}
if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
(or_flags &
(VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
| VNET_BUFFER_F_OFFLOAD_IP_CKSUM
| VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
(or_flags & VNET_BUFFER_F_OFFLOAD)))
{
dpdk_buffer_tx_offload (xd, b[0], mb[0]);
dpdk_buffer_tx_offload (xd, b[1], mb[1]);
@ -388,10 +388,7 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm,
}
if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
(or_flags &
(VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
| VNET_BUFFER_F_OFFLOAD_IP_CKSUM
| VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
(or_flags & VNET_BUFFER_F_OFFLOAD)))
{
dpdk_buffer_tx_offload (xd, b[0], mb[0]);
dpdk_buffer_tx_offload (xd, b[1], mb[1]);

14
src/plugins/gtpu/gtpu_decap.c
View File

@ -1257,13 +1257,15 @@ static char *gtpu_flow_error_strings[] = {
};
#define gtpu_local_need_csum_check(_b) \
(!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED \
|| _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
#define gtpu_local_need_csum_check(_b) \
(!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED || \
(_b->flags & VNET_BUFFER_F_OFFLOAD && \
vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
#define gtpu_local_csum_is_valid(_b) \
((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT \
|| _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) != 0)
#define gtpu_local_csum_is_valid(_b) \
((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT || \
(_b->flags & VNET_BUFFER_F_OFFLOAD && \
vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0)
static_always_inline u8
gtpu_validate_udp_csum (vlib_main_t * vm, vlib_buffer_t *b)

2
src/plugins/lisp/lisp-cp/packets.c
View File

@ -182,7 +182,7 @@ pkt_push_udp_and_ip (vlib_main_t * vm, vlib_buffer_t * b, u16 sp, u16 dp,
if (csum_offload)
{
ih = pkt_push_ip (vm, b, sip, dip, IP_PROTOCOL_UDP, 1);
b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
vnet_buffer (b)->l3_hdr_offset = (u8 *) ih - b->data;
vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data;
uh->checksum = 0;

21
src/plugins/vmxnet3/input.c
View File

@ -79,6 +79,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
u16 gso_size)
{
u8 l4_hdr_sz = 0;
u32 oflags = 0;
if (rx_comp->flags & VMXNET3_RXCF_IP4)
{
@ -90,15 +91,15 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) +
ip4_header_bytes (ip4);
hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4;
VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4;
/* checksum offload */
if (!(rx_comp->index & VMXNET3_RXCI_CNC))
{
if (!(rx_comp->flags & VMXNET3_RXCF_IPC))
{
hb->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
ip4->checksum = 0;
}
if (!(rx_comp->flags & VMXNET3_RXCF_TUC))
@ -108,7 +109,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
tcp_header_t *tcp =
(tcp_header_t *) (hb->data +
vnet_buffer (hb)->l4_hdr_offset);
hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp->checksum = 0;
}
else if (rx_comp->flags & VMXNET3_RXCF_UDP)
@ -116,7 +117,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
udp_header_t *udp =
(udp_header_t *) (hb->data +
vnet_buffer (hb)->l4_hdr_offset);
hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp->checksum = 0;
}
}
@ -148,8 +149,8 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) +
sizeof (ip6_header_t);
hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6;
VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6;
/* checksum offload */
if (!(rx_comp->index & VMXNET3_RXCI_CNC))
@ -161,7 +162,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
tcp_header_t *tcp =
(tcp_header_t *) (hb->data +
vnet_buffer (hb)->l4_hdr_offset);
hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp->checksum = 0;
}
else if (rx_comp->flags & VMXNET3_RXCF_UDP)
@ -169,7 +170,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
udp_header_t *udp =
(udp_header_t *) (hb->data +
vnet_buffer (hb)->l4_hdr_offset);
hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp->checksum = 0;
}
}
@ -194,6 +195,8 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
hb->flags |= VNET_BUFFER_F_GSO;
}
}
if (oflags)
vnet_buffer_offload_flags_set (hb, oflags);
}
static_always_inline uword

3
src/plugins/wireguard/wireguard_input.c
View File

@ -368,7 +368,8 @@ VLIB_NODE_FN (wg_input_node) (vlib_main_t * vm,
clib_memcpy (vlib_buffer_get_current (b[0]), decr_data, decr_len);
b[0]->current_length = decr_len;
b[0]->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
vnet_buffer_offload_flags_clear (b[0],
VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
wg_timers_any_authenticated_packet_received (peer);
wg_timers_any_authenticated_packet_traversal (peer);

20
src/vnet/buffer.c
View File

@ -16,6 +16,18 @@
#include <vlib/vlib.h>
#include <vnet/buffer.h>
u8 *
format_vnet_buffer_offload (u8 *s, va_list *args)
{
vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
#define _(bit, name, ss, v) \
if (v && (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name)) \
s = format (s, "%s ", ss);
foreach_vnet_buffer_offload_flag
#undef _
return s;
}
u8 *
format_vnet_buffer (u8 * s, va_list * args)
@ -29,7 +41,10 @@ format_vnet_buffer (u8 * s, va_list * args)
a = format (a, "%s ", ss);
foreach_vnet_buffer_flag
#undef _
if (b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID)
if (b->flags & VNET_BUFFER_F_OFFLOAD) a =
format (a, "%U ", format_vnet_buffer_offload, b);
if (b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID)
a = format (a, "l2-hdr-offset %d ", vnet_buffer (b)->l2_hdr_offset);
if (b->flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID)
@ -38,6 +53,9 @@ format_vnet_buffer (u8 * s, va_list * args)
if (b->flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID)
a = format (a, "l4-hdr-offset %d ", vnet_buffer (b)->l4_hdr_offset);
if (b->flags & VNET_BUFFER_F_GSO)
a = format (a, "gso gso-size %d", vnet_buffer2 (b)->gso_size);
if (b->flags & VNET_BUFFER_F_QOS_DATA_VALID)
a = format (a, "qos %d.%d ",
vnet_buffer2 (b)->qos.bits, vnet_buffer2 (b)->qos.source);

108
src/vnet/buffer.h
View File

@ -46,34 +46,34 @@
* Flags that are set in the high order bits of ((vlib_buffer*)b)->flags
*
*/
#define foreach_vnet_buffer_flag \
_( 1, L4_CHECKSUM_COMPUTED, "l4-cksum-computed", 1) \
_( 2, L4_CHECKSUM_CORRECT, "l4-cksum-correct", 1) \
_( 3, VLAN_2_DEEP, "vlan-2-deep", 1) \
_( 4, VLAN_1_DEEP, "vlan-1-deep", 1) \
_( 5, SPAN_CLONE, "span-clone", 1) \
_( 6, LOOP_COUNTER_VALID, "loop-counter-valid", 0) \
_( 7, LOCALLY_ORIGINATED, "local", 1) \
_( 8, IS_IP4, "ip4", 1) \
_( 9, IS_IP6, "ip6", 1) \
_(10, OFFLOAD_IP_CKSUM, "offload-ip-cksum", 1) \
_(11, OFFLOAD_TCP_CKSUM, "offload-tcp-cksum", 1) \
_(12, OFFLOAD_UDP_CKSUM, "offload-udp-cksum", 1) \
_(13, IS_NATED, "natted", 1) \
_(14, L2_HDR_OFFSET_VALID, "l2_hdr_offset_valid", 0) \
_(15, L3_HDR_OFFSET_VALID, "l3_hdr_offset_valid", 0) \
_(16, L4_HDR_OFFSET_VALID, "l4_hdr_offset_valid", 0) \
_(17, FLOW_REPORT, "flow-report", 1) \
_(18, IS_DVR, "dvr", 1) \
_(19, QOS_DATA_VALID, "qos-data-valid", 0) \
_(20, GSO, "gso", 0) \
_(21, AVAIL1, "avail1", 1) \
_(22, AVAIL2, "avail2", 1) \
_(23, AVAIL3, "avail3", 1) \
_(24, AVAIL4, "avail4", 1) \
_(25, AVAIL5, "avail5", 1) \
_(26, AVAIL6, "avail6", 1) \
_(27, AVAIL7, "avail7", 1)
#define foreach_vnet_buffer_flag \
_ (1, L4_CHECKSUM_COMPUTED, "l4-cksum-computed", 1) \
_ (2, L4_CHECKSUM_CORRECT, "l4-cksum-correct", 1) \
_ (3, VLAN_2_DEEP, "vlan-2-deep", 1) \
_ (4, VLAN_1_DEEP, "vlan-1-deep", 1) \
_ (5, SPAN_CLONE, "span-clone", 1) \
_ (6, LOOP_COUNTER_VALID, "loop-counter-valid", 0) \
_ (7, LOCALLY_ORIGINATED, "local", 1) \
_ (8, IS_IP4, "ip4", 1) \
_ (9, IS_IP6, "ip6", 1) \
_ (10, OFFLOAD, "offload", 0) \
_ (11, IS_NATED, "natted", 1) \
_ (12, L2_HDR_OFFSET_VALID, "l2_hdr_offset_valid", 0) \
_ (13, L3_HDR_OFFSET_VALID, "l3_hdr_offset_valid", 0) \
_ (14, L4_HDR_OFFSET_VALID, "l4_hdr_offset_valid", 0) \
_ (15, FLOW_REPORT, "flow-report", 1) \
_ (16, IS_DVR, "dvr", 1) \
_ (17, QOS_DATA_VALID, "qos-data-valid", 0) \
_ (18, GSO, "gso", 0) \
_ (19, AVAIL1, "avail1", 1) \
_ (20, AVAIL2, "avail2", 1) \
_ (21, AVAIL3, "avail3", 1) \
_ (22, AVAIL4, "avail4", 1) \
_ (23, AVAIL5, "avail5", 1) \
_ (24, AVAIL6, "avail6", 1) \
_ (25, AVAIL7, "avail7", 1) \
_ (26, AVAIL8, "avail8", 1) \
_ (27, AVAIL9, "avail9", 1)
/*
* Please allocate the FIRST available bit, redefine
@ -81,10 +81,10 @@
* VNET_BUFFER_FLAGS_ALL_AVAIL definition.
*/
#define VNET_BUFFER_FLAGS_ALL_AVAIL \
(VNET_BUFFER_F_AVAIL1 | VNET_BUFFER_F_AVAIL2 | VNET_BUFFER_F_AVAIL3 | \
VNET_BUFFER_F_AVAIL4 | VNET_BUFFER_F_AVAIL5 | VNET_BUFFER_F_AVAIL6 | \
VNET_BUFFER_F_AVAIL7)
#define VNET_BUFFER_FLAGS_ALL_AVAIL \
(VNET_BUFFER_F_AVAIL1 | VNET_BUFFER_F_AVAIL2 | VNET_BUFFER_F_AVAIL3 | \
VNET_BUFFER_F_AVAIL4 | VNET_BUFFER_F_AVAIL5 | VNET_BUFFER_F_AVAIL6 | \
VNET_BUFFER_F_AVAIL7 | VNET_BUFFER_F_AVAIL8 | VNET_BUFFER_F_AVAIL9)
#define VNET_BUFFER_FLAGS_VLAN_BITS \
(VNET_BUFFER_F_VLAN_1_DEEP | VNET_BUFFER_F_VLAN_2_DEEP)
@ -420,6 +420,21 @@ STATIC_ASSERT (sizeof (vnet_buffer_opaque_t) <=
#define vnet_buffer(b) ((vnet_buffer_opaque_t *) (b)->opaque)
#define foreach_vnet_buffer_offload_flag \
_ (0, IP_CKSUM, "offload-ip-cksum", 1) \
_ (1, TCP_CKSUM, "offload-tcp-cksum", 1) \
_ (2, UDP_CKSUM, "offload-udp-cksum", 1) \
_ (3, OUTER_IP_CKSUM, "offload-outer-ip-cksum", 1) \
_ (4, OUTER_TCP_CKSUM, "offload-outer-tcp-cksum", 1) \
_ (5, OUTER_UDP_CKSUM, "offload-outer-udp-cksum", 1)
enum
{
#define _(bit, name, s, v) VNET_BUFFER_OFFLOAD_F_##name = (1 << bit),
foreach_vnet_buffer_offload_flag
#undef _
};
/* Full cache line (64 bytes) of additional space */
typedef struct
{
@ -452,12 +467,15 @@ typedef struct
* in case the egress interface is not GSO-enabled - then we need to perform
* the segmentation, and use this value to cut the payload appropriately.
*/
u16 gso_size;
/* size of L4 prototol header */
u16 gso_l4_hdr_sz;
struct
{
u16 gso_size;
/* size of L4 prototol header */
u16 gso_l4_hdr_sz;
/* The union below has a u64 alignment, so this space is unused */
u32 __unused2[1];
/* offload flags */
u32 oflags;
};
struct
{
@ -501,6 +519,22 @@ STATIC_ASSERT (sizeof (vnet_buffer_opaque2_t) <=
format_function_t format_vnet_buffer;
format_function_t format_vnet_buffer_offload;
static_always_inline void
vnet_buffer_offload_flags_set (vlib_buffer_t *b, u32 oflags)
{
vnet_buffer2 (b)->oflags |= oflags;
b->flags |= VNET_BUFFER_F_OFFLOAD;
}
static_always_inline void
vnet_buffer_offload_flags_clear (vlib_buffer_t *b, u32 oflags)
{
vnet_buffer2 (b)->oflags &= ~oflags;
if (0 == vnet_buffer2 (b)->oflags)
b->flags &= ~VNET_BUFFER_F_OFFLOAD;
}
#endif /* included_vnet_buffer_h */

13
src/vnet/devices/af_packet/node.c
View File

@ -120,6 +120,7 @@ static_always_inline void
mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
{
ethernet_header_t *eth = vlib_buffer_get_current (b);
u32 oflags = 0;
if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP4)
{
ip4_header_t *ip4 =
@ -127,7 +128,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
b->flags |= VNET_BUFFER_F_IS_IP4;
if (ip4->protocol == IP_PROTOCOL_TCP)
{
b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) +
ip4_header_bytes (ip4));
@ -136,7 +137,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
}
else if (ip4->protocol == IP_PROTOCOL_UDP)
{
b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) +
ip4_header_bytes (ip4));
@ -146,6 +147,8 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t);
vnet_buffer (b)->l4_hdr_offset =
sizeof (ethernet_header_t) + ip4_header_bytes (ip4);
if (oflags)
vnet_buffer_offload_flags_set (b, oflags);
}
else if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP6)
{
@ -165,7 +168,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
}
if (ip6->protocol == IP_PROTOCOL_TCP)
{
b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp =
(tcp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) + ip6_hdr_len);
@ -174,7 +177,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
}
else if (ip6->protocol == IP_PROTOCOL_UDP)
{
b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp =
(udp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) + ip6_hdr_len);
@ -184,6 +187,8 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t);
vnet_buffer (b)->l4_hdr_offset =
sizeof (ethernet_header_t) + ip6_hdr_len;
if (oflags)
vnet_buffer_offload_flags_set (b, oflags);
}
}

22
src/vnet/devices/virtio/device.c
View File

@ -282,6 +282,8 @@ static_always_inline void
set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
const int is_l2)
{
u32 oflags = vnet_buffer2 (b)->oflags;
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
ip4_header_t *ip4;
@ -290,11 +292,11 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
0 /* ip6 */ );
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset; // 0x22;
if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
}
else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
}
@ -305,7 +307,7 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
*/
ip4 =
(ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
}
else if (b->flags & VNET_BUFFER_F_IS_IP6)
@ -315,11 +317,11 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
1 /* ip6 */ );
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset; // 0x36;
if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
}
else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
}
@ -330,6 +332,8 @@ static_always_inline void
set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
const int is_l2)
{
u32 oflags = vnet_buffer2 (b)->oflags;
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
ip4_header_t *ip4;
@ -348,7 +352,7 @@ set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
* virtio devices do not support IP4 checksum offload. So driver takes care
* of it while doing tx.
*/
if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
}
else if (b->flags & VNET_BUFFER_F_IS_IP6)
@ -392,8 +396,7 @@ add_buffer_to_slot (vlib_main_t * vm, vlib_node_runtime_t * node,
goto done;
}
}
else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
else if (b->flags & VNET_BUFFER_F_OFFLOAD)
{
if (csum_offload)
set_checksum_offsets (b, hdr, is_l2);
@ -584,8 +587,7 @@ add_buffer_to_slot_packed (vlib_main_t * vm, vlib_node_runtime_t * node,
goto done;
}
}
else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
else if (b->flags & VNET_BUFFER_F_OFFLOAD)
{
if (csum_offload)
set_checksum_offsets (b, hdr, is_l2);

14
src/vnet/devices/virtio/node.c
View File

@ -235,6 +235,7 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
{
u16 ethertype = 0, l2hdr_sz = 0;
u32 oflags = 0;
if (type == VIRTIO_IF_TYPE_TUN)
{
@ -280,11 +281,10 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
(ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
*l4_proto = ip4->protocol;
oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
b0->flags |=
(VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
b0->flags |=
(VNET_BUFFER_F_L2_HDR_OFFSET_VALID
| VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
(VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
@ -301,7 +301,7 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
}
if (*l4_proto == IP_PROTOCOL_TCP)
{
b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b0) +
vnet_buffer
(b0)->l4_hdr_offset);
@ -309,12 +309,14 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
}
else if (*l4_proto == IP_PROTOCOL_UDP)
{
b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b0) +
vnet_buffer
(b0)->l4_hdr_offset);
*l4_hdr_sz = sizeof (*udp);
}
if (oflags)
vnet_buffer_offload_flags_set (b0, oflags);
}
}

11
src/vnet/devices/virtio/vhost_user_input.c
View File

@ -253,6 +253,7 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
ethernet_header_t *eh = (ethernet_header_t *) b0_data;
u16 ethertype = clib_net_to_host_u16 (eh->type);
u16 l2hdr_sz = sizeof (ethernet_header_t);
u32 oflags = 0;
if (ethernet_frame_is_tagged (ethertype))
{
@ -278,7 +279,8 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
{
ip4_header_t *ip4 = (ip4_header_t *) (b0_data + l2hdr_sz);
l4_proto = ip4->protocol;
b0->flags |= VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
b0->flags |= VNET_BUFFER_F_IS_IP4;
oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
{
@ -292,12 +294,12 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
tcp_header_t *tcp = (tcp_header_t *)
(b0_data + vnet_buffer (b0)->l4_hdr_offset);
l4_hdr_sz = tcp_header_bytes (tcp);
b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
}
else if (l4_proto == IP_PROTOCOL_UDP)
{
l4_hdr_sz = sizeof (udp_header_t);
b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
}
if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP)
@ -318,6 +320,9 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
}
if (oflags)
vnet_buffer_offload_flags_set (b0, oflags);
}
static_always_inline void

19
src/vnet/devices/virtio/vhost_user_output.c
View File

@ -225,10 +225,11 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
generic_header_offset_t gho = { 0 };
int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
u32 oflags = vnet_buffer2 (b)->oflags;
ASSERT (!(is_ip4 && is_ip6));
vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6);
if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
{
ip4_header_t *ip4;
@ -238,13 +239,13 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
}
/* checksum offload */
if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset;
hdr->csum_offset = offsetof (udp_header_t, checksum);
}
else if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset;
@ -254,7 +255,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
/* GSO offload */
if (b->flags & VNET_BUFFER_F_GSO)
{
if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
if (is_ip4 &&
(vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)))
@ -270,7 +271,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
}
}
else if ((vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)) &&
(b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
(oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
{
hdr->gso_size = vnet_buffer2 (b)->gso_size;
hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
@ -477,9 +478,7 @@ retry:
hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
hdr->num_buffers = 1;
or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
(b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
(b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
/* Guest supports csum offload and buffer requires checksum offload? */
if (or_flags &&
@ -813,9 +812,7 @@ retry:
hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
hdr->num_buffers = 1; //This is local, no need to check
or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
(b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
(b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
/* Guest supports csum offload and buffer requires checksum offload? */
if (or_flags

13
src/vnet/gso/gro_func.h
View File

@ -147,7 +147,7 @@ gro_validate_checksum (vlib_main_t * vm, vlib_buffer_t * b0,
{
u32 flags = 0;
if (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
if (b0->flags & VNET_BUFFER_F_OFFLOAD)
return VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
vlib_buffer_advance (b0, gho0->l3_hdr_offset);
if (is_ip4)
@ -353,9 +353,9 @@ gro_fixup_header (vlib_main_t * vm, vlib_buffer_t * b0, u32 ack_number,
ip4->length =
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
gho0.l3_hdr_offset);
b0->flags |=
(VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4 |
VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4);
vnet_buffer_offload_flags_set (b0, (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
}
else if (gho0.gho_flags & GHO_F_IP6)
{
@ -364,9 +364,8 @@ gro_fixup_header (vlib_main_t * vm, vlib_buffer_t * b0, u32 ack_number,
ip6->payload_length =
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
gho0.l4_hdr_offset);
b0->flags |=
(VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6 |
VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
vnet_buffer_offload_flags_set (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
}
tcp_header_t *tcp0 =

10
src/vnet/gso/node.c
View File

@ -143,7 +143,8 @@ tso_segment_vxlan_tunnel_headers_fixup (vlib_main_t * vm, vlib_buffer_t * b,
{
udp->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
}
b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
/* FIXME: it should be OUTER_UDP_CKSUM */
vnet_buffer_offload_flags_clear (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
}
}
@ -281,7 +282,8 @@ tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags,
tcp->checksum = 0;
tcp->checksum =
ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6, &bogus);
b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
vnet_buffer_offload_flags_clear (b0,
VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
}
}
else
@ -295,8 +297,8 @@ tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags,
tcp->checksum = 0;
tcp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip4);
}
b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
}
if (!is_l2 && ((gho->gho_flags & GHO_F_TUNNEL) == 0))

28
src/vnet/interface_format.c
View File

@ -722,7 +722,7 @@ unformat_vnet_buffer_flags (unformat_input_t * input, va_list * args)
while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
{
/* Red herring, there is no such buffer flag */
if (unformat (input, "avail8"))
if (unformat (input, "avail10"))
return 0;
#define _(bit,enum,str,verbose) \
else if (unformat (input, str)) \
@ -740,6 +740,32 @@ unformat_vnet_buffer_flags (unformat_input_t * input, va_list * args)
return rv;
}
uword
unformat_vnet_buffer_offload_flags (unformat_input_t *input, va_list *args)
{
u32 *flagp = va_arg (*args, u32 *);
int rv = 0;
u32 oflags = 0;
while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
{
if (0)
;
#define _(bit, enum, str, verbose) \
else if (unformat (input, str)) \
{ \
oflags |= (1 << bit); \
rv = 1; \
}
foreach_vnet_buffer_offload_flag
#undef _
else break;
}
if (rv)
*flagp = oflags;
return rv;
}
uword
unformat_vnet_hw_interface (unformat_input_t * input, va_list * args)
{

1
src/vnet/interface_funcs.h
View File

@ -457,6 +457,7 @@ format_function_t format_vnet_sw_interface_flags;
unformat_function_t unformat_vnet_sw_interface;
unformat_function_t unformat_vnet_hw_interface;
unformat_function_t unformat_vnet_buffer_flags;
unformat_function_t unformat_vnet_buffer_offload_flags;
/* Parses interface flags (up, down, enable, disable, etc.) */
unformat_function_t unformat_vnet_hw_interface_flags;

19
src/vnet/interface_output.c
View File

@ -337,28 +337,24 @@ vnet_interface_output_node_inline (vlib_main_t * vm,
if (do_tx_offloads)
{
u32 vnet_buffer_offload_flags =
(VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
if (or_flags & vnet_buffer_offload_flags)
if (or_flags & VNET_BUFFER_F_OFFLOAD)
{
if (b[0]->flags & vnet_buffer_offload_flags)
if (b[0]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[0],
b[0]->flags & VNET_BUFFER_F_IS_IP4,
b[0]->flags & VNET_BUFFER_F_IS_IP6);
if (b[1]->flags & vnet_buffer_offload_flags)
if (b[1]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[1],
b[1]->flags & VNET_BUFFER_F_IS_IP4,
b[1]->flags & VNET_BUFFER_F_IS_IP6);
if (b[2]->flags & vnet_buffer_offload_flags)
if (b[2]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[2],
b[2]->flags & VNET_BUFFER_F_IS_IP4,
b[2]->flags & VNET_BUFFER_F_IS_IP6);
if (b[3]->flags & vnet_buffer_offload_flags)
if (b[3]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[3],
b[3]->flags & VNET_BUFFER_F_IS_IP4,
@ -406,10 +402,7 @@ vnet_interface_output_node_inline (vlib_main_t * vm,
if (do_tx_offloads)
{
if (b[0]->flags &
(VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
VNET_BUFFER_F_OFFLOAD_IP_CKSUM))
if (b[0]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[0],
b[0]->flags & VNET_BUFFER_F_IS_IP4,

31
src/vnet/interface_output.h
View File

@ -44,18 +44,17 @@
#include <vnet/gso/hdr_offset_parser.h>
static_always_inline void
vnet_calc_ip4_checksums (vlib_main_t * vm, vlib_buffer_t * b,
ip4_header_t * ip4, tcp_header_t * th,
udp_header_t * uh)
vnet_calc_ip4_checksums (vlib_main_t *vm, vlib_buffer_t *b, ip4_header_t *ip4,
tcp_header_t *th, udp_header_t *uh, u32 oflags)
{
if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
th->checksum = 0;
th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
}
if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
uh->checksum = 0;
uh->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
@ -63,17 +62,16 @@ vnet_calc_ip4_checksums (vlib_main_t * vm, vlib_buffer_t * b,
}
static_always_inline void
vnet_calc_ip6_checksums (vlib_main_t * vm, vlib_buffer_t * b,
ip6_header_t * ip6, tcp_header_t * th,
udp_header_t * uh)
vnet_calc_ip6_checksums (vlib_main_t *vm, vlib_buffer_t *b, ip6_header_t *ip6,
tcp_header_t *th, udp_header_t *uh, u32 oflags)
{
int bogus;
if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
th->checksum = 0;
th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
}
if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
uh->checksum = 0;
uh->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
@ -88,6 +86,7 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b,
ip6_header_t *ip6;
tcp_header_t *th;
udp_header_t *uh;
u32 oflags = vnet_buffer2 (b)->oflags;
ASSERT (!(is_ip4 && is_ip6));
@ -98,16 +97,16 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b,
if (is_ip4)
{
vnet_calc_ip4_checksums (vm, b, ip4, th, uh);
vnet_calc_ip4_checksums (vm, b, ip4, th, uh, oflags);
}
else if (is_ip6)
{
vnet_calc_ip6_checksums (vm, b, ip6, th, uh);
vnet_calc_ip6_checksums (vm, b, ip6, th, uh, oflags);
}
b->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
vnet_buffer_offload_flags_clear (b, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
VNET_BUFFER_OFFLOAD_F_UDP_CKSUM |
VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
}
#endif

9
src/vnet/ip/ip4_forward.c
View File

@ -1417,9 +1417,10 @@ ip4_local_l4_csum_validate (vlib_main_t * vm, vlib_buffer_t * p,
}
}
#define ip4_local_csum_is_offloaded(_b) \
_b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM \
|| _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM
#define ip4_local_csum_is_offloaded(_b) \
((_b->flags & VNET_BUFFER_F_OFFLOAD) && \
(vnet_buffer2 (_b)->oflags & \
(VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
#define ip4_local_need_csum_check(is_tcp_udp, _b) \
(is_tcp_udp && !(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED \
@ -2066,7 +2067,7 @@ ip4_ttl_and_checksum_check (vlib_buffer_t * b, ip4_header_t * ip, u16 * next,
/* Verify checksum. */
ASSERT (ip4_header_checksum_is_valid (ip) ||
(b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
(vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
}

2
src/vnet/ip/ip4_inlines.h
View File

@ -125,7 +125,7 @@ vlib_buffer_push_ip4_custom (vlib_main_t * vm, vlib_buffer_t * b,
if (csum_offload)
{
ih->checksum = 0;
b->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_IP_CKSUM);
}
else
ih->checksum = ip4_header_checksum (ih);

62
src/vnet/ip/ip6_forward.c
View File

@ -1333,15 +1333,23 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
flags[0] = b[0]->flags;
flags[1] = b[1]->flags;
u32 oflags[2];
oflags[0] = vnet_buffer2 (b[0])->oflags;
oflags[1] = vnet_buffer2 (b[1])->oflags;
u32 l4_offload[2];
l4_offload[0] = (flags[0] & VNET_BUFFER_F_OFFLOAD) &&
(oflags[0] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
l4_offload[1] = (flags[1] & VNET_BUFFER_F_OFFLOAD) &&
(oflags[1] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
u32 good_l4_csum[2];
good_l4_csum[0] =
flags[0] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
(flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[0];
good_l4_csum[1] =
flags[1] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
(flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[1];
u32 udp_offset[2] = { };
u8 is_tcp_udp[2];
@ -1515,11 +1523,15 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
u8 type = lm->builtin_protocol_by_ip_protocol[ip->protocol];
u32 flags = b[0]->flags;
u32 good_l4_csum =
flags & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
u32 oflags = vnet_buffer2 (b[0])->oflags;
u32 l4_offload = (flags & VNET_BUFFER_F_OFFLOAD) &&
(oflags & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
u32 good_l4_csum =
(flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload;
u32 udp_offset;
i16 len_diff = 0;
u8 is_tcp_udp = ip6_next_proto_is_tcp_udp (b[0], ip, &udp_offset);
@ -1539,8 +1551,9 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UNKNOWN;
len_diff = type == IP_BUILTIN_PROTOCOL_UDP ? len_diff : 0;
u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN && !good_l4_csum
&& !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN &&
!good_l4_csum &&
!(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
if (PREDICT_FALSE (need_csum))
{
flags = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
@ -1556,7 +1569,6 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
error = len_diff < 0 ? IP6_ERROR_UDP_LENGTH : error;
STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
IP6_ERROR_UDP_CHECKSUM,
"Wrong IP6 errors constants");
@ -1882,8 +1894,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
if (p1->flags & VNET_BUFFER_F_GSO)
ip1_len = gso_mtu_sz (p1);
ip6_mtu_check (p0, ip0_len,
adj0[0].rewrite_header.max_l3_packet_bytes,
is_locally_originated0, &next0, is_midchain,
@ -1892,18 +1902,15 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
adj1[0].rewrite_header.max_l3_packet_bytes,
is_locally_originated1, &next1, is_midchain,
&error1);
/* Don't adjust the buffer for hop count issue; icmp-error node
* wants to see the IP header */
if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
{
p0->current_data -= rw_len0;
p0->current_length += rw_len0;
tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
next0 = adj0[0].rewrite_header.next_index;
if (PREDICT_FALSE
(adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
vnet_feature_arc_start_w_cfg_index
@ -2070,7 +2077,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
adj0[0].rewrite_header.max_l3_packet_bytes,
is_locally_originated0, &next0, is_midchain,
&error0);
/* Don't adjust the buffer for hop count issue; icmp-error node
* wants to see the IP header */
if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
@ -2289,7 +2295,6 @@ format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
ip6_hop_by_hop_option_t *opt0, *limit0;
ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
u8 type0;
s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d",
hbh0->protocol, (hbh0->length + 1) << 3, total_len);
@ -2312,9 +2317,8 @@ format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
}
else
{
s =
format (s, "\n unrecognized option %d length %d", type0,
opt0->length);
s = format (s, "\n unrecognized option %d length %d", type0,
opt0->length);
}
opt0 =
(ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
@ -2362,9 +2366,8 @@ format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
}
else
{
s =
format (s, "\n unrecognized option %d length %d", type0,
opt0->length);
s = format (s, "\n unrecognized option %d length %d", type0,
opt0->length);
}
opt0 =
(ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
@ -3012,8 +3015,7 @@ VLIB_CLI_COMMAND (show_ip6_local, static) =
#ifndef CLIB_MARCH_VARIANT
int
vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
u32 table_index)
vnet_set_ip6_classify_intfc (vlib_main_t *vm, u32 sw_if_index, u32 table_index)
{
vnet_main_t *vnm = vnet_get_main ();
vnet_interface_main_t *im = &vnm->interface_main;
@ -3044,17 +3046,13 @@ vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
sw_if_index);
if (table_index != (u32) ~ 0)
{
dpo_id_t dpo = DPO_INVALID;
dpo_set (&dpo,
DPO_CLASSIFY,
DPO_PROTO_IP6,
classify_dpo_create (DPO_PROTO_IP6, table_index));
fib_table_entry_special_dpo_add (fib_index,
&pfx,
FIB_SOURCE_CLASSIFY,

2
src/vnet/ip/ip_frag.c
View File

@ -226,7 +226,7 @@ ip4_frag_do_fragment (vlib_main_t * vm, u32 from_bi, u16 mtu,
to_ip4->checksum = ip4_header_checksum (to_ip4);
/* we've just done the IP checksum .. */
to_b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
vnet_buffer_offload_flags_clear (to_b, VNET_BUFFER_OFFLOAD_F_IP_CKSUM);
rem -= len;
fo += len;

75
src/vnet/ipsec/ipsec_output.c
View File

@ -309,45 +309,50 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
next_node_index = im->ah4_encrypt_node_index;
vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
if (is_ipv6)
if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_OFFLOAD))
{
if (PREDICT_FALSE
(b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM))
u32 oflags = vnet_buffer2 (b0)->oflags;
/*
* Clearing offload flags before checksum is computed
* It guarantees the cache hit!
*/
vnet_buffer_offload_flags_clear (b0, oflags);
if (is_ipv6)
{
tcp0->checksum =
ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0,
&bogus);
b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
if (PREDICT_FALSE (oflags &
VNET_BUFFER_OFFLOAD_F_TCP_CKSUM))
{
tcp0->checksum = ip6_tcp_udp_icmp_compute_checksum (
vm, b0, ip6_0, &bogus);
}
if (PREDICT_FALSE (oflags &
VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
{
udp0->checksum = ip6_tcp_udp_icmp_compute_checksum (
vm, b0, ip6_0, &bogus);
}
}
if (PREDICT_FALSE
(b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
else
{
udp0->checksum =
ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0,
&bogus);
b0->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
}
}
else
{
if (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
{
ip0->checksum = ip4_header_checksum (ip0);
b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
}
if (PREDICT_FALSE
(b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM))
{
tcp0->checksum =
ip4_tcp_udp_compute_checksum (vm, b0, ip0);
b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
}
if (PREDICT_FALSE
(b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
{
udp0->checksum =
ip4_tcp_udp_compute_checksum (vm, b0, ip0);
b0->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
if (PREDICT_FALSE (oflags &
VNET_BUFFER_OFFLOAD_F_IP_CKSUM))
{
ip0->checksum = ip4_header_checksum (ip0);
}
if (PREDICT_FALSE (oflags &
VNET_BUFFER_OFFLOAD_F_TCP_CKSUM))
{
tcp0->checksum =
ip4_tcp_udp_compute_checksum (vm, b0, ip0);
}
if (PREDICT_FALSE (oflags &
VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
{
udp0->checksum =
ip4_tcp_udp_compute_checksum (vm, b0, ip0);
}
}
}
vlib_buffer_advance (b0, iph_offset);

4
src/vnet/pg/cli.c
View File

@ -379,7 +379,9 @@ new_stream (vlib_main_t * vm,
else if (unformat (input, "buffer-flags %U",
unformat_vnet_buffer_flags, &s.buffer_flags))
;
else if (unformat (input, "buffer-offload-flags %U",
unformat_vnet_buffer_offload_flags, &s.buffer_oflags))
;
else if (unformat (input, "node %U",
unformat_vlib_node, vm, &s.node_index))
;

31
src/vnet/pg/input.c
View File

@ -1537,13 +1537,14 @@ pg_input_trace (pg_main_t * pg,
}
static_always_inline void
fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
int gso_enabled, u32 gso_size)
fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers,
u32 buffer_oflags, int gso_enabled, u32 gso_size)
{
for (int i = 0; i < n_buffers; i++)
{
vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[i]);
u8 l4_proto = 0;
u32 oflags = 0;
ethernet_header_t *eh =
(ethernet_header_t *) vlib_buffer_get_current (b0);
@ -1574,10 +1575,11 @@ fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
l4_proto = ip4->protocol;
b0->flags |=
(VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
b0->flags |= (VNET_BUFFER_F_L2_HDR_OFFSET_VALID
| VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
(VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
{
@ -1594,7 +1596,8 @@ fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
if (l4_proto == IP_PROTOCOL_TCP)
{
b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
/* only set GSO flag for chained buffers */
if (gso_enabled && (b0->flags & VLIB_BUFFER_NEXT_PRESENT))
@ -1609,8 +1612,12 @@ fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
}
else if (l4_proto == IP_PROTOCOL_UDP)
{
b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
}
if (oflags)
vnet_buffer_offload_flags_set (b0, oflags);
}
}
@ -1710,13 +1717,11 @@ pg_generate_packets (vlib_node_runtime_t * node,
vnet_buffer (b)->feature_arc_index = feature_arc_index;
}
if (pi->gso_enabled ||
(s->buffer_flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
VNET_BUFFER_F_OFFLOAD_IP_CKSUM)))
if (pi->gso_enabled || (s->buffer_flags & VNET_BUFFER_F_OFFLOAD))
{
fill_buffer_offload_flags (vm, to_next, n_this_frame,
pi->gso_enabled, pi->gso_size);
s->buffer_oflags, pi->gso_enabled,
pi->gso_size);
}
n_trace = vlib_get_trace_count (vm, node);

5
src/vnet/pg/pg.h
View File

@ -125,9 +125,12 @@ typedef struct pg_stream_t
for max_packet_bytes. */
u32 buffer_bytes;
/* Buffer flags to set in each packet e.g. checksum offload flags */
/* Buffer flags to set in each packet e.g. l2 valid flags */
u32 buffer_flags;
/* Buffer offload flags to set in each packet e.g. checksum offload flags */
u32 buffer_oflags;
/* Last packet length if packet size edit type is increment. */
u32 last_increment_packet_size;

2
src/vnet/tcp/tcp_output.c
View File

@ -430,7 +430,7 @@ tcp_compute_checksum (tcp_connection_t * tc, vlib_buffer_t * b)
}
else
{
b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
}
return checksum;
}

2
src/vnet/udp/udp_inlines.h
View File

@ -35,7 +35,7 @@ vlib_buffer_push_udp (vlib_buffer_t * b, u16 sp, u16 dp, u8 offload_csum)
uh->checksum = 0;
uh->length = clib_host_to_net_u16 (udp_len);
if (offload_csum)
b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data;
b->flags |= VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
return uh;

Some files were not shown because too many files have changed in this diff Show More