ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vlib: add buffer cloning support

Browse Source 
Change-Id: I50070611af15b2b4cc29664a8bee4f821ac3c835
Signed-off-by: Damjan Marion <damarion@cisco.com>
This commit is contained in:
Damjan Marion
2017-01-25 14:18:03 +01:00
committed by Dave Barach
parent 05472b625f
commit c47ed032c6
8 changed files with 254 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/scripts/vnet/mcast/ip4
View File

@ -2,7 +2,7 @@ packet-generator new {
name x
limit 1
node ip4-input
size 64-64
size 512-512
no-recycle
data {
ICMP: 1.0.0.2 -> 232.1.1.1
@ -11,12 +11,15 @@ packet-generator new {
}
}
trace add pg-input 100
loop create
loop create
set int state loop0 up
set int state loop1 up
create packet-generator interface pg1
create packet-generator interface pg2
create packet-generator interface pg3
set int state pg1 up
set int state pg2 up
set int state pg3 up
ip mroute add 232.1.1.1 via pg0 Accept
ip mroute add 232.1.1.1 via loop0 Forward
ip mroute add 232.1.1.1 via loop1 Forward
ip mroute add 232.1.1.1 via pg1 Forward
ip mroute add 232.1.1.1 via pg2 Forward
ip mroute add 232.1.1.1 via pg3 Forward

248
src/vlib/buffer.c
View File

File diff suppressed because it is too large Load Diff

4
src/vlib/buffer.h
View File

@ -119,7 +119,9 @@ typedef struct
feature node
*/
u8 dont_waste_me[3]; /**< Available space in the (precious)
u8 n_add_refs; /**< Number of additional references to this buffer. */
u8 dont_waste_me[2]; /**< Available space in the (precious)
first 32 octets of buffer metadata
Before allocating any of it, discussion required!
*/

113
src/vlib/buffer_funcs.h
View File

@ -530,6 +530,110 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b)
return fd;
}
/** \brief Create multiple clones of buffer and store them in the supplied array
@param vm - (vlib_main_t *) vlib main data structure pointer
@param src_buffer - (u32) source buffer index
@param buffers - (u32 * ) buffer index array
@param n_buffers - (u8) number of buffer clones requested
@param head_end_offset - (u16) offset relative to current position
where packet head ends
@return - (u8) number of buffers actually cloned, may be
less than the number requested or zero
*/
always_inline u8
vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
u8 n_buffers, u16 head_end_offset)
{
u8 i;
vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
ASSERT (s->n_add_refs == 0);
ASSERT (n_buffers);
if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
{
buffers[0] = src_buffer;
for (i = 1; i < n_buffers; i++)
{
vlib_buffer_t *d;
d = vlib_buffer_copy (vm, s);
if (d == 0)
return i;
buffers[i] = vlib_get_buffer_index (vm, d);
}
return n_buffers;
}
n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
s->free_list_index);
if (PREDICT_FALSE (n_buffers == 0))
{
buffers[0] = src_buffer;
return 1;
}
for (i = 0; i < n_buffers; i++)
{
vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
d->current_data = s->current_data;
d->current_length = head_end_offset;
d->free_list_index = s->free_list_index;
d->total_length_not_including_first_buffer =
s->total_length_not_including_first_buffer + s->current_length -
head_end_offset;
d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
clib_memcpy (vlib_buffer_get_current (d), vlib_buffer_get_current (s),
head_end_offset);
d->next_buffer = src_buffer;
}
vlib_buffer_advance (s, head_end_offset);
s->n_add_refs = n_buffers - 1;
while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
{
s = vlib_get_buffer (vm, s->next_buffer);
s->n_add_refs = n_buffers - 1;
}
return n_buffers;
}
/** \brief Attach cloned tail to the buffer
@param vm - (vlib_main_t *) vlib main data structure pointer
@param head - (vlib_buffer_t *) head buffer
@param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
*/
always_inline void
vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head,
vlib_buffer_t * tail)
{
ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
ASSERT (head->free_list_index == tail->free_list_index);
head->flags |= VLIB_BUFFER_NEXT_PRESENT;
head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
head->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
head->next_buffer = vlib_get_buffer_index (vm, tail);
head->total_length_not_including_first_buffer = tail->current_length +
tail->total_length_not_including_first_buffer;
next_segment:
__sync_add_and_fetch (&tail->n_add_refs, 1);
if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
{
tail = vlib_get_buffer (vm, tail->next_buffer);
goto next_segment;
}
}
/* Initializes the buffer as an empty packet with no chained buffers. */
always_inline void
vlib_buffer_chain_init (vlib_buffer_t * first)
@ -695,7 +799,8 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst,
_(flags);
_(free_list_index);
#undef _
ASSERT (dst->total_length_not_including_first_buffer == 0);
dst->total_length_not_including_first_buffer = 0;
ASSERT (dst->n_add_refs == 0);
}
always_inline void
@ -727,8 +832,10 @@ vlib_buffer_init_two_for_free_list (vlib_buffer_t * dst0,
_(flags);
_(free_list_index);
#undef _
ASSERT (dst0->total_length_not_including_first_buffer == 0);
ASSERT (dst1->total_length_not_including_first_buffer == 0);
dst0->total_length_not_including_first_buffer = 0;
dst1->total_length_not_including_first_buffer = 0;
ASSERT (dst0->n_add_refs == 0);
ASSERT (dst1->n_add_refs == 0);
}
#if CLIB_DEBUG > 0

41
src/vnet/devices/dpdk/buffer.c
View File

@ -79,20 +79,46 @@
STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
"VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
static_always_inline void
dpdk_rte_pktmbuf_free (vlib_main_t * vm, vlib_buffer_t * b)
{
vlib_buffer_t *hb = b;
struct rte_mbuf *mb;
u32 next, flags;
mb = rte_mbuf_from_vlib_buffer (hb);
next:
flags = b->flags;
next = b->next_buffer;
mb = rte_mbuf_from_vlib_buffer (b);
if (PREDICT_FALSE (b->n_add_refs))
{
rte_mbuf_refcnt_update (mb, b->n_add_refs);
b->n_add_refs = 0;
}
rte_pktmbuf_free_seg (mb);
if (flags & VLIB_BUFFER_NEXT_PRESENT)
{
b = vlib_get_buffer (vm, next);
goto next;
}
}
static void
del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
{
u32 i;
struct rte_mbuf *mb;
vlib_buffer_t *b;
for (i = 0; i < vec_len (f->buffers); i++)
{
b = vlib_get_buffer (vm, f->buffers[i]);
mb = rte_mbuf_from_vlib_buffer (b);
ASSERT (rte_mbuf_refcnt_read (mb) == 1);
rte_pktmbuf_free (mb);
dpdk_rte_pktmbuf_free (vm, b);
}
vec_free (f->name);
vec_free (f->buffers);
}
@ -325,7 +351,6 @@ vlib_buffer_free_inline (vlib_main_t * vm,
for (i = 0; i < n_buffers; i++)
{
vlib_buffer_t *b;
struct rte_mbuf *mb;
b = vlib_get_buffer (vm, buffers[i]);
@ -351,11 +376,7 @@ vlib_buffer_free_inline (vlib_main_t * vm,
else
{
if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
{
mb = rte_mbuf_from_vlib_buffer (b);
ASSERT (rte_mbuf_refcnt_read (mb) == 1);
rte_pktmbuf_free (mb);
}
dpdk_rte_pktmbuf_free (vm, b);
}
}
if (vec_len (bm->announce_list))

11
src/vnet/devices/dpdk/device.c
View File

@ -168,13 +168,11 @@ dpdk_validate_rte_mbuf (vlib_main_t * vm, vlib_buffer_t * b,
{
b2 = vlib_get_buffer (vm, b2->next_buffer);
mb = rte_mbuf_from_vlib_buffer (b2);
last_mb->next = mb;
last_mb = mb;
rte_pktmbuf_reset (mb);
}
}
first_mb = mb = rte_mbuf_from_vlib_buffer (b);
last_mb = first_mb = mb = rte_mbuf_from_vlib_buffer (b);
first_mb->nb_segs = 1;
mb->data_len = b->current_length;
mb->pkt_len = maybe_multiseg ? vlib_buffer_length_in_chain (vm, b) :
@ -185,10 +183,17 @@ dpdk_validate_rte_mbuf (vlib_main_t * vm, vlib_buffer_t * b,
{
b = vlib_get_buffer (vm, b->next_buffer);
mb = rte_mbuf_from_vlib_buffer (b);
last_mb->next = mb;
last_mb = mb;
mb->data_len = b->current_length;
mb->pkt_len = b->current_length;
mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
first_mb->nb_segs++;
if (PREDICT_FALSE (b->n_add_refs))
{
rte_mbuf_refcnt_update (mb, b->n_add_refs);
b->n_add_refs = 0;
}
}
}

78
src/vnet/dpo/replicate_dpo.c
View File

@ -625,6 +625,7 @@ replicate_inline (vlib_main_t * vm,
vlib_frame_t * frame)
{
vlib_combined_counter_main_t * cm = &replicate_main.repm_counters;
replicate_main_t * rm = &replicate_main;
u32 n_left_from, * from, * to_next, next_index;
u32 cpu_index = os_get_cpu_number();
@ -645,13 +646,11 @@ replicate_inline (vlib_main_t * vm,
const replicate_t *rep0;
vlib_buffer_t * b0, *c0;
const dpo_id_t *dpo0;
u8 num_cloned;
bi0 = from[0];
to_next[0] = bi0;
from += 1;
to_next += 1;
n_left_from -= 1;
n_left_to_next -= 1;
b0 = vlib_get_buffer (vm, bi0);
repi0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
@ -661,50 +660,21 @@ replicate_inline (vlib_main_t * vm,
cm, cpu_index, repi0, 1,
vlib_buffer_length_in_chain(vm, b0));
/* ship the original to the first bucket */
dpo0 = replicate_get_bucket_i(rep0, 0);
next0 = dpo0->dpoi_next_node;
vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
vec_validate (rm->clones[cpu_index], rep0->rep_n_buckets - 1);
if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
num_cloned = vlib_buffer_clone (vm, bi0, rm->clones[cpu_index], rep0->rep_n_buckets, 128);
if (num_cloned != rep0->rep_n_buckets)
{
vlib_node_increment_counter
(vm, node->node_index,
REPLICATE_DPO_ERROR_BUFFER_ALLOCATION_FAILURE, 1);
}
for (bucket = 0; bucket < num_cloned; bucket++)
{
replicate_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
t->rep_index = repi0;
t->dpo = *dpo0;
}
vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
to_next, n_left_to_next,
bi0, next0);
/* ship copies to the rest of the buckets */
for (bucket = 1; bucket < rep0->rep_n_buckets; bucket++)
{
/*
* After the enqueue of the first buffer, and of all subsequent
* buffers in this loop, it is possible that we over-flow the
* frame of the to-next node. When this happens we need to 'put'
* that full frame to the node and get a fresh empty one.
* Note that these are macros with side effects that change
* to_next & n_left_to_next
*/
if (PREDICT_FALSE(0 == n_left_to_next))
{
vlib_put_next_frame (vm, node, next_index, n_left_to_next);
vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
}
/* Make a copy. This can fail, so deal with it. */
c0 = vlib_buffer_copy(vm, b0);
if (PREDICT_FALSE (c0 == 0))
{
vlib_node_increment_counter
(vm, node->node_index,
REPLICATE_DPO_ERROR_BUFFER_ALLOCATION_FAILURE,
1);
continue;
}
ci0 = vlib_get_buffer_index(vm, c0);
ci0 = rm->clones[cpu_index][bucket];
c0 = vlib_get_buffer(vm, ci0);
to_next[0] = ci0;
to_next += 1;
@ -724,7 +694,13 @@ replicate_inline (vlib_main_t * vm,
vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
to_next, n_left_to_next,
ci0, next0);
if (PREDICT_FALSE (n_left_to_next == 0))
{
vlib_put_next_frame (vm, node, next_index, n_left_to_next);
vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
}
}
vec_reset_length (rm->clones[cpu_index]);
}
vlib_put_next_frame (vm, node, next_index, n_left_to_next);
@ -797,3 +773,15 @@ VLIB_REGISTER_NODE (ip6_replicate_node) = {
[0] = "error-drop",
},
};
clib_error_t *
replicate_dpo_init (vlib_main_t * vm)
{
replicate_main_t * rm = &replicate_main;
vec_validate (rm->clones, vlib_num_workers());
return 0;
}
VLIB_INIT_FUNCTION (replicate_dpo_init);

3
src/vnet/dpo/replicate_dpo.h
View File

@ -32,6 +32,9 @@
typedef struct replicate_main_t_
{
vlib_combined_counter_main_t repm_counters;
/* per-cpu vector of cloned packets */
u32 **clones;
} replicate_main_t;
extern replicate_main_t replicate_main;