Type: improvement
Change-Id: Idef30aee80b654ce424b6f1f1f730574ca68874c
Signed-off-by: Damjan Marion <damarion@cisco.com>
memif_copy_op_t *co;
memif_region_index_t last_region = ~0;
void *last_region_shm = 0;
memif_copy_op_t *co;
memif_region_index_t last_region = ~0;
void *last_region_shm = 0;
ring = mq->ring;
ring_size = 1 << mq->log2_ring_size;
ring = mq->ring;
ring_size = 1 << mq->log2_ring_size;
- free_slots = ring->tail - mq->last_tail;
- mq->last_tail += free_slots;
- slot = (type == MEMIF_RING_S2M) ? ring->head : ring->tail;
-
if (type == MEMIF_RING_S2M)
if (type == MEMIF_RING_S2M)
- free_slots = ring_size - ring->head + mq->last_tail;
+ {
+ slot = head = ring->head;
+ tail = __atomic_load_n (&ring->tail, __ATOMIC_ACQUIRE);
+ mq->last_tail += tail - mq->last_tail;
+ free_slots = ring_size - head + mq->last_tail;
+ }
- free_slots = ring->head - ring->tail;
+ {
+ slot = tail = ring->tail;
+ head = __atomic_load_n (&ring->head, __ATOMIC_ACQUIRE);
+ mq->last_tail += tail - mq->last_tail;
+ free_slots = head - tail;
+ }
while (n_left && free_slots)
{
while (n_left && free_slots)
{
vec_reset_length (ptd->copy_ops);
vec_reset_length (ptd->buffers);
vec_reset_length (ptd->copy_ops);
vec_reset_length (ptd->buffers);
- CLIB_MEMORY_STORE_BARRIER ();
if (type == MEMIF_RING_S2M)
if (type == MEMIF_RING_S2M)
+ __atomic_store_n (&ring->head, slot, __ATOMIC_RELEASE);
+ __atomic_store_n (&ring->tail, slot, __ATOMIC_RELEASE);
if (n_left && n_retries--)
goto retry;
if (n_left && n_retries--)
goto retry;
u16 mask = ring_size - 1;
int n_retries = 5;
vlib_buffer_t *b0;
u16 mask = ring_size - 1;
int n_retries = 5;
vlib_buffer_t *b0;
- n_free = ring->tail - mq->last_tail;
+ slot = tail = __atomic_load_n (&ring->tail, __ATOMIC_ACQUIRE);
+ head = ring->head;
+
+ n_free = tail - mq->last_tail;
if (n_free >= 16)
{
vlib_buffer_free_from_ring_no_next (vm, mq->buffers,
if (n_free >= 16)
{
vlib_buffer_free_from_ring_no_next (vm, mq->buffers,
mq->last_tail += n_free;
}
mq->last_tail += n_free;
}
- slot = ring->head;
- free_slots = ring_size - ring->head + mq->last_tail;
+ free_slots = ring_size - head + mq->last_tail;
while (n_left && free_slots)
{
while (n_left && free_slots)
{
- CLIB_MEMORY_STORE_BARRIER ();
- ring->head = slot;
+ __atomic_store_n (&ring->head, slot, __ATOMIC_RELEASE);
if (n_left && n_retries--)
goto retry;
if (n_left && n_retries--)
goto retry;
/* for S2M rings, we are consumers of packet buffers, and for M2S rings we
are producers of empty buffers */
cur_slot = (type == MEMIF_RING_S2M) ? mq->last_head : mq->last_tail;
/* for S2M rings, we are consumers of packet buffers, and for M2S rings we
are producers of empty buffers */
cur_slot = (type == MEMIF_RING_S2M) ? mq->last_head : mq->last_tail;
- last_slot = (type == MEMIF_RING_S2M) ? ring->head : ring->tail;
+
+ if (type == MEMIF_RING_S2M)
+ last_slot = __atomic_load_n (&ring->head, __ATOMIC_ACQUIRE);
+ else
+ last_slot = __atomic_load_n (&ring->tail, __ATOMIC_ACQUIRE);
+
if (cur_slot == last_slot)
goto refill;
n_slots = last_slot - cur_slot;
if (cur_slot == last_slot)
goto refill;
n_slots = last_slot - cur_slot;
/* release slots from the ring */
if (type == MEMIF_RING_S2M)
{
/* release slots from the ring */
if (type == MEMIF_RING_S2M)
{
- CLIB_MEMORY_STORE_BARRIER ();
- ring->tail = mq->last_head = cur_slot;
+ __atomic_store_n (&ring->tail, cur_slot, __ATOMIC_RELEASE);
+ mq->last_head = cur_slot;
d->length = mif->run.buffer_size;
}
d->length = mif->run.buffer_size;
}
- CLIB_MEMORY_STORE_BARRIER ();
- ring->head = head;
+ __atomic_store_n (&ring->head, head, __ATOMIC_RELEASE);
buffer_length = vlib_buffer_get_default_data_size (vm) - start_offset;
cur_slot = mq->last_tail;
buffer_length = vlib_buffer_get_default_data_size (vm) - start_offset;
cur_slot = mq->last_tail;
- last_slot = ring->tail;
+ last_slot = __atomic_load_n (&ring->tail, __ATOMIC_ACQUIRE);
if (cur_slot == last_slot)
goto refill;
n_slots = last_slot - cur_slot;
if (cur_slot == last_slot)
goto refill;
n_slots = last_slot - cur_slot;
- CLIB_MEMORY_STORE_BARRIER ();
- ring->head = head;
+ __atomic_store_n (&ring->head, head, __ATOMIC_RELEASE);
done:
return n_rx_packets;
done:
return n_rx_packets;