ligang/vpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

avf: descriptor should be volatile

Browse Source 
Change-Id: I2cb4cf2167b6e958d2e57b461848a4a189e3fda0
Signed-off-by: Damjan Marion <damarion@cisco.com>
This commit is contained in:
Damjan Marion
2018-07-11 13:29:17 +02:00
committed by Damjan Marion
parent 151b322492
commit 9a02c54e4f
4 changed files with 60 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/plugins/avf/avf.h
View File

@ -38,19 +38,39 @@ enum
#undef _ #undef _
}; };
typedef struct typedef volatile struct
{ {
u64 qword[4]; union
{
struct
{
u64 mirr:13;
u64 rsv1:3;
u64 l2tag1:16;
u64 filter_status:32;
u64 status:19;
u64 error:8;
u64 rsv2:3;
u64 ptype:8;
u64 length:26;
};
u64 qword[4];
#ifdef CLIB_HAVE_VEC256
u64x4 as_u64x4;
#endif
};
} avf_rx_desc_t; } avf_rx_desc_t;
STATIC_ASSERT_SIZEOF (avf_rx_desc_t, 32); STATIC_ASSERT_SIZEOF (avf_rx_desc_t, 32);
typedef struct typedef volatile struct
{ {
union union
{ {
u64 qword[2]; u64 qword[2];
#ifdef CLIB_HAVE_VEC128
u64x2 as_u64x2; u64x2 as_u64x2;
#endif
}; };
} avf_tx_desc_t; } avf_tx_desc_t;
@ -64,7 +84,7 @@ typedef struct
u16 size; u16 size;
avf_rx_desc_t *descs; avf_rx_desc_t *descs;
u32 *bufs; u32 *bufs;
u16 n_bufs; u16 n_enqueued;
} avf_rxq_t; } avf_rxq_t;
typedef struct typedef struct
@ -76,7 +96,7 @@ typedef struct
clib_spinlock_t lock; clib_spinlock_t lock;
avf_tx_desc_t *descs; avf_tx_desc_t *descs;
u32 *bufs; u32 *bufs;
u16 n_bufs; u16 n_enqueued;
} avf_txq_t; } avf_txq_t;
typedef struct typedef struct

32
src/plugins/avf/device.c
View File

@ -227,7 +227,7 @@ avf_rxq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 rxq_size)
rxq->descs = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error, rxq->descs = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error,
rxq->size * sizeof (avf_rx_desc_t), rxq->size * sizeof (avf_rx_desc_t),
2 * CLIB_CACHE_LINE_BYTES); 2 * CLIB_CACHE_LINE_BYTES);
memset (rxq->descs, 0, rxq->size * sizeof (avf_rx_desc_t)); memset ((void *) rxq->descs, 0, rxq->size * sizeof (avf_rx_desc_t));
vec_validate_aligned (rxq->bufs, rxq->size, CLIB_CACHE_LINE_BYTES); vec_validate_aligned (rxq->bufs, rxq->size, CLIB_CACHE_LINE_BYTES);
rxq->qrx_tail = ad->bar0 + AVF_QRX_TAIL (qid); rxq->qrx_tail = ad->bar0 + AVF_QRX_TAIL (qid);
@ -236,7 +236,7 @@ avf_rxq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 rxq_size)
if (n_alloc == 0) if (n_alloc == 0)
return clib_error_return (0, "buffer allocation error"); return clib_error_return (0, "buffer allocation error");
rxq->n_bufs = n_alloc; rxq->n_enqueued = n_alloc;
avf_rx_desc_t *d = rxq->descs; avf_rx_desc_t *d = rxq->descs;
for (i = 0; i < n_alloc; i++) for (i = 0; i < n_alloc; i++)
{ {
@ -323,7 +323,7 @@ avf_adminq_init (vlib_main_t * vm, avf_device_t * ad)
pa = avf_dma_addr (vm, ad, ad->atq); pa = avf_dma_addr (vm, ad, ad->atq);
avf_reg_write (ad, AVF_ATQT, 0); /* Tail */ avf_reg_write (ad, AVF_ATQT, 0); /* Tail */
avf_reg_write (ad, AVF_ATQH, 0); /* Head */ avf_reg_write (ad, AVF_ATQH, 0); /* Head */
avf_reg_write (ad, AVF_ATQLEN, AVF_MBOX_LEN | (1 << 31)); /* len & ena */ avf_reg_write (ad, AVF_ATQLEN, AVF_MBOX_LEN | (1ULL << 31)); /* len & ena */
avf_reg_write (ad, AVF_ATQBAL, (u32) pa); /* Base Address Low */ avf_reg_write (ad, AVF_ATQBAL, (u32) pa); /* Base Address Low */
avf_reg_write (ad, AVF_ATQBAH, (u32) (pa >> 32)); /* Base Address High */ avf_reg_write (ad, AVF_ATQBAH, (u32) (pa >> 32)); /* Base Address High */
@ -338,7 +338,7 @@ avf_adminq_init (vlib_main_t * vm, avf_device_t * ad)
avf_reg_write (ad, AVF_ARQH, 0); /* Head */ avf_reg_write (ad, AVF_ARQH, 0); /* Head */
avf_reg_write (ad, AVF_ARQT, 0); /* Head */ avf_reg_write (ad, AVF_ARQT, 0); /* Head */
avf_reg_write (ad, AVF_ARQLEN, AVF_MBOX_LEN | (1 << 31)); /* len & ena */ avf_reg_write (ad, AVF_ARQLEN, AVF_MBOX_LEN | (1ULL << 31)); /* len & ena */
avf_reg_write (ad, AVF_ARQBAL, (u32) pa); /* Base Address Low */ avf_reg_write (ad, AVF_ARQBAL, (u32) pa); /* Base Address Low */
avf_reg_write (ad, AVF_ARQBAH, (u32) (pa >> 32)); /* Base Address High */ avf_reg_write (ad, AVF_ARQBAH, (u32) (pa >> 32)); /* Base Address High */
avf_reg_write (ad, AVF_ARQT, AVF_MBOX_LEN - 1); /* Tail */ avf_reg_write (ad, AVF_ARQT, AVF_MBOX_LEN - 1); /* Tail */
@ -552,7 +552,7 @@ avf_op_config_vsi_queues (vlib_main_t * vm, avf_device_t * ad)
avf_rxq_t *q = vec_elt_at_index (ad->rxqs, i); avf_rxq_t *q = vec_elt_at_index (ad->rxqs, i);
rxq->ring_len = q->size; rxq->ring_len = q->size;
rxq->databuffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES; rxq->databuffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES;
rxq->dma_ring_addr = avf_dma_addr (vm, ad, q->descs); rxq->dma_ring_addr = avf_dma_addr (vm, ad, (void *) q->descs);
avf_reg_write (ad, AVF_QRX_TAIL (i), q->size - 1); avf_reg_write (ad, AVF_QRX_TAIL (i), q->size - 1);
} }
@ -562,7 +562,7 @@ avf_op_config_vsi_queues (vlib_main_t * vm, avf_device_t * ad)
{ {
txq->queue_id = i; txq->queue_id = i;
txq->ring_len = q->size; txq->ring_len = q->size;
txq->dma_ring_addr = avf_dma_addr (vm, ad, q->descs); txq->dma_ring_addr = avf_dma_addr (vm, ad, (void *) q->descs);
} }
} }
@ -618,7 +618,7 @@ avf_op_enable_queues (vlib_main_t * vm, avf_device_t * ad, u32 rx, u32 tx)
qs.rx_queues = rx; qs.rx_queues = rx;
qs.tx_queues = tx; qs.tx_queues = tx;
avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, 0); avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, 0);
avf_reg_write (ad, AVF_QRX_TAIL (0), rxq->n_bufs); avf_reg_write (ad, AVF_QRX_TAIL (0), rxq->n_enqueued);
return avf_send_to_pf (vm, ad, VIRTCHNL_OP_ENABLE_QUEUES, &qs, return avf_send_to_pf (vm, ad, VIRTCHNL_OP_ENABLE_QUEUES, &qs,
sizeof (virtchnl_queue_select_t), 0, 0); sizeof (virtchnl_queue_select_t), 0, 0);
} }
@ -769,14 +769,14 @@ avf_process_one_device (vlib_main_t * vm, avf_device_t * ad, int is_irq)
ASSERT (ad->error == 0); ASSERT (ad->error == 0);
r = avf_get_u32 (ad->bar0, AVF_ARQLEN); r = avf_get_u32 (ad->bar0, AVF_ARQLEN);
if ((r & 0xf0000000) != (1 << 31)) if ((r & 0xf0000000) != (1ULL << 31))
{ {
ad->error = clib_error_return (0, "arq not enabled, arqlen = 0x%x", r); ad->error = clib_error_return (0, "arq not enabled, arqlen = 0x%x", r);
goto error; goto error;
} }
r = avf_get_u32 (ad->bar0, AVF_ATQLEN); r = avf_get_u32 (ad->bar0, AVF_ATQLEN);
if ((r & 0xf0000000) != (1 << 31)) if ((r & 0xf0000000) != (1ULL << 31))
{ {
ad->error = clib_error_return (0, "atq not enabled, atqlen = 0x%x", r); ad->error = clib_error_return (0, "atq not enabled, atqlen = 0x%x", r);
goto error; goto error;
@ -1032,10 +1032,10 @@ avf_delete_if (vlib_main_t * vm, avf_device_t * ad)
vec_foreach_index (i, ad->rxqs) vec_foreach_index (i, ad->rxqs)
{ {
avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i); avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i);
vlib_physmem_free (vm, am->physmem_region, rxq->descs); vlib_physmem_free (vm, am->physmem_region, (void *) rxq->descs);
if (rxq->n_bufs) if (rxq->n_enqueued)
vlib_buffer_free_from_ring (vm, rxq->bufs, rxq->next, rxq->size, vlib_buffer_free_from_ring (vm, rxq->bufs, rxq->next, rxq->size,
rxq->n_bufs); rxq->n_enqueued);
vec_free (rxq->bufs); vec_free (rxq->bufs);
} }
/* *INDENT-ON* */ /* *INDENT-ON* */
@ -1045,12 +1045,12 @@ avf_delete_if (vlib_main_t * vm, avf_device_t * ad)
vec_foreach_index (i, ad->txqs) vec_foreach_index (i, ad->txqs)
{ {
avf_txq_t *txq = vec_elt_at_index (ad->txqs, i); avf_txq_t *txq = vec_elt_at_index (ad->txqs, i);
vlib_physmem_free (vm, am->physmem_region, txq->descs); vlib_physmem_free (vm, am->physmem_region, (void *) txq->descs);
if (txq->n_bufs) if (txq->n_enqueued)
{ {
u16 first = (txq->next - txq->n_bufs) & (txq->size -1); u16 first = (txq->next - txq->n_enqueued) & (txq->size -1);
vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size, vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size,
txq->n_bufs); txq->n_enqueued);
} }
vec_free (txq->bufs); vec_free (txq->bufs);
} }

26
src/plugins/avf/input.c
View File

@ -54,7 +54,7 @@ avf_rxq_refill (vlib_main_t * vm, vlib_node_runtime_t * node, avf_rxq_t * rxq,
u32 s0, s1, s2, s3; u32 s0, s1, s2, s3;
avf_rx_desc_t *d[4]; avf_rx_desc_t *d[4];
n_refill = rxq->size - 1 - rxq->n_bufs; n_refill = rxq->size - 1 - rxq->n_enqueued;
if (PREDICT_TRUE (n_refill <= AVF_INPUT_REFILL_TRESHOLD)) if (PREDICT_TRUE (n_refill <= AVF_INPUT_REFILL_TRESHOLD))
return; return;
@ -74,7 +74,7 @@ avf_rxq_refill (vlib_main_t * vm, vlib_node_runtime_t * node, avf_rxq_t * rxq,
return; return;
} }
rxq->n_bufs += n_alloc; rxq->n_enqueued += n_alloc;
while (n_alloc >= 4) while (n_alloc >= 4)
{ {
@ -355,13 +355,13 @@ avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
if (rxq->next + 11 < rxq->size) if (rxq->next + 11 < rxq->size)
{ {
int stride = 8; int stride = 8;
CLIB_PREFETCH (rxq->descs + (rxq->next + stride), CLIB_PREFETCH ((void *) (rxq->descs + (rxq->next + stride)),
CLIB_CACHE_LINE_BYTES, LOAD); CLIB_CACHE_LINE_BYTES, LOAD);
CLIB_PREFETCH (rxq->descs + (rxq->next + stride + 1), CLIB_PREFETCH ((void *) (rxq->descs + (rxq->next + stride + 1)),
CLIB_CACHE_LINE_BYTES, LOAD); CLIB_CACHE_LINE_BYTES, LOAD);
CLIB_PREFETCH (rxq->descs + (rxq->next + stride + 2), CLIB_PREFETCH ((void *) (rxq->descs + (rxq->next + stride + 2)),
CLIB_CACHE_LINE_BYTES, LOAD); CLIB_CACHE_LINE_BYTES, LOAD);
CLIB_PREFETCH (rxq->descs + (rxq->next + stride + 3), CLIB_PREFETCH ((void *) (rxq->descs + (rxq->next + stride + 3)),
CLIB_CACHE_LINE_BYTES, LOAD); CLIB_CACHE_LINE_BYTES, LOAD);
} }
@ -404,21 +404,21 @@ avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
rxq->next = (rxq->next + 4) & mask; rxq->next = (rxq->next + 4) & mask;
d = rxq->descs + rxq->next; d = rxq->descs + rxq->next;
n_rxv += 4; n_rxv += 4;
rxq->n_bufs -= 4; rxq->n_enqueued -= 4;
bi += 4; bi += 4;
continue; continue;
one_by_one: one_by_one:
#endif #endif
CLIB_PREFETCH (rxq->descs + ((rxq->next + 8) & mask), CLIB_PREFETCH ((void *) (rxq->descs + ((rxq->next + 8) & mask)),
CLIB_CACHE_LINE_BYTES, LOAD); CLIB_CACHE_LINE_BYTES, LOAD);
if ((d->qword[1] & AVF_RX_DESC_STATUS_DD) == 0) if ((d->qword[1] & AVF_RX_DESC_STATUS_DD) == 0)
break; break;
rxve = ptd->rx_vector + n_rxv; rxve = ptd->rx_vector + n_rxv;
bi[0] = rxq->bufs[rxq->next]; bi[0] = rxq->bufs[rxq->next];
rxve->status = avf_get_u64_bits (d, 8, 18, 0); rxve->status = avf_get_u64_bits ((void *) d, 8, 18, 0);
rxve->error = avf_get_u64_bits (d, 8, 26, 19); rxve->error = avf_get_u64_bits ((void *) d, 8, 26, 19);
rxve->ptype = avf_get_u64_bits (d, 8, 37, 30); rxve->ptype = avf_get_u64_bits ((void *) d, 8, 37, 30);
rxve->length = avf_get_u64_bits (d, 8, 63, 38); rxve->length = avf_get_u64_bits ((void *) d, 8, 63, 38);
maybe_error |= rxve->error; maybe_error |= rxve->error;
/* deal with chained buffers */ /* deal with chained buffers */
@ -431,7 +431,7 @@ avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
rxq->next = (rxq->next + 1) & mask; rxq->next = (rxq->next + 1) & mask;
d = rxq->descs + rxq->next; d = rxq->descs + rxq->next;
n_rxv++; n_rxv++;
rxq->n_bufs--; rxq->n_enqueued--;
bi++; bi++;
} }

12
src/plugins/avf/output.c
View File

@ -57,12 +57,12 @@ CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm,
clib_spinlock_lock_if_init (&txq->lock); clib_spinlock_lock_if_init (&txq->lock);
/* release cosumed bufs */ /* release cosumed bufs */
if (txq->n_bufs) if (txq->n_enqueued)
{ {
u16 first, slot, n_free = 0; u16 first, slot, n_free = 0;
first = slot = (txq->next - txq->n_bufs) & mask; first = slot = (txq->next - txq->n_enqueued) & mask;
d0 = txq->descs + slot; d0 = txq->descs + slot;
while (n_free < txq->n_bufs && avf_tx_desc_get_dtyp (d0) == 0x0F) while (n_free < txq->n_enqueued && avf_tx_desc_get_dtyp (d0) == 0x0F)
{ {
n_free++; n_free++;
slot = (slot + 1) & mask; slot = (slot + 1) & mask;
@ -71,7 +71,7 @@ CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm,
if (n_free) if (n_free)
{ {
txq->n_bufs -= n_free;; txq->n_enqueued -= n_free;
vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size, vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size,
n_free); n_free);
} }
@ -126,7 +126,7 @@ CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm,
d3->qword[1] = ((u64) b3->current_length) << 34 | bits; d3->qword[1] = ((u64) b3->current_length) << 34 | bits;
txq->next = (txq->next + 4) & mask; txq->next = (txq->next + 4) & mask;
txq->n_bufs += 4; txq->n_enqueued += 4;
buffers += 4; buffers += 4;
n_left -= 4; n_left -= 4;
} }
@ -147,7 +147,7 @@ CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm,
d0->qword[1] = (((u64) b0->current_length) << 34) | bits; d0->qword[1] = (((u64) b0->current_length) << 34) | bits;
txq->next = (txq->next + 1) & mask; txq->next = (txq->next + 1) & mask;
txq->n_bufs++; txq->n_enqueued++;
buffers++; buffers++;
n_left--; n_left--;
} }