#include <memif/memif.h>
#include <memif/private.h>
-#define foreach_memif_tx_func_error \
-_(NO_FREE_SLOTS, "no free tx slots") \
-_(ROLLBACK, "no enough space in tx buffers")
+#define foreach_memif_tx_func_error \
+ _ (NO_FREE_SLOTS, no_free_slots, ERROR, "no free tx slots") \
+ _ (ROLLBACK, rollback, ERROR, "no enough space in tx buffers")
typedef enum
{
-#define _(f,s) MEMIF_TX_ERROR_##f,
+#define _(f, n, s, d) MEMIF_TX_ERROR_##f,
foreach_memif_tx_func_error
#undef _
MEMIF_TX_N_ERROR,
} memif_tx_func_error_t;
-static char *memif_tx_func_error_strings[] = {
-#define _(n,s) s,
+static vlib_error_desc_t memif_tx_func_error_counters[] = {
+#define _(f, n, s, d) { #n, d, VL_COUNTER_SEVERITY_##s },
foreach_memif_tx_func_error
#undef _
};
}
static_always_inline uword
-memif_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
- vlib_frame_t * frame, memif_if_t * mif,
- memif_ring_type_t type, memif_queue_t * mq,
- memif_per_thread_data_t * ptd)
+memif_interface_tx_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
+ u32 *buffers, memif_if_t *mif,
+ memif_ring_type_t type, memif_queue_t *mq,
+ memif_per_thread_data_t *ptd, u32 n_left)
{
memif_ring_t *ring;
- u32 *buffers = vlib_frame_vector_args (frame);
- u32 n_left = frame->n_vectors;
u32 n_copy_op;
u16 ring_size, mask, slot, free_slots;
int n_retries = 5;
memif_copy_op_t *co;
memif_region_index_t last_region = ~0;
void *last_region_shm = 0;
+ u16 head, tail;
ring = mq->ring;
ring_size = 1 << mq->log2_ring_size;
retry:
- free_slots = ring->tail - mq->last_tail;
- mq->last_tail += free_slots;
- slot = (type == MEMIF_RING_S2M) ? ring->head : ring->tail;
-
if (type == MEMIF_RING_S2M)
- free_slots = ring_size - ring->head + mq->last_tail;
+ {
+ slot = head = ring->head;
+ tail = __atomic_load_n (&ring->tail, __ATOMIC_ACQUIRE);
+ mq->last_tail += tail - mq->last_tail;
+ free_slots = ring_size - head + mq->last_tail;
+ }
else
- free_slots = ring->head - ring->tail;
+ {
+ slot = tail = ring->tail;
+ head = __atomic_load_n (&ring->head, __ATOMIC_ACQUIRE);
+ mq->last_tail += tail - mq->last_tail;
+ free_slots = head - tail;
+ }
while (n_left && free_slots)
{
u32 saved_ptd_buffers_len = _vec_len (ptd->buffers);
u16 saved_slot = slot;
- CLIB_PREFETCH (&ring->desc[(slot + 8) & mask], CLIB_CACHE_LINE_BYTES,
- LOAD);
+ clib_prefetch_load (&ring->desc[(slot + 8) & mask]);
d0 = &ring->desc[slot & mask];
if (PREDICT_FALSE (last_region != d0->region))
{
slot++;
free_slots--;
+ d0->length = dst_off;
d0->flags = MEMIF_DESC_FLAG_NEXT;
d0 = &ring->desc[slot & mask];
dst_off = 0;
co = ptd->copy_ops;
while (n_copy_op >= 8)
{
- CLIB_PREFETCH (co[4].data, CLIB_CACHE_LINE_BYTES, LOAD);
- CLIB_PREFETCH (co[5].data, CLIB_CACHE_LINE_BYTES, LOAD);
- CLIB_PREFETCH (co[6].data, CLIB_CACHE_LINE_BYTES, LOAD);
- CLIB_PREFETCH (co[7].data, CLIB_CACHE_LINE_BYTES, LOAD);
+ clib_prefetch_load (co[4].data);
+ clib_prefetch_load (co[5].data);
+ clib_prefetch_load (co[6].data);
+ clib_prefetch_load (co[7].data);
b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]);
b1 = vlib_get_buffer (vm, ptd->buffers[co[1].buffer_vec_index]);
vec_reset_length (ptd->copy_ops);
vec_reset_length (ptd->buffers);
- CLIB_MEMORY_STORE_BARRIER ();
if (type == MEMIF_RING_S2M)
- ring->head = slot;
+ __atomic_store_n (&ring->head, slot, __ATOMIC_RELEASE);
else
- ring->tail = slot;
+ __atomic_store_n (&ring->tail, slot, __ATOMIC_RELEASE);
if (n_left && n_retries--)
goto retry;
- clib_spinlock_unlock_if_init (&mif->lockp);
-
- if (n_left)
- {
- vlib_error_count (vm, node->node_index, MEMIF_TX_ERROR_NO_FREE_SLOTS,
- n_left);
- }
-
- if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0 && mq->int_fd > -1)
- {
- u64 b = 1;
- CLIB_UNUSED (int r) = write (mq->int_fd, &b, sizeof (b));
- mq->int_count++;
- }
-
- vlib_buffer_free (vm, vlib_frame_vector_args (frame), frame->n_vectors);
-
- return frame->n_vectors;
+ return n_left;
}
static_always_inline uword
-memif_interface_tx_zc_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
- vlib_frame_t * frame, memif_if_t * mif,
- memif_queue_t * mq,
- memif_per_thread_data_t * ptd)
+memif_interface_tx_zc_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
+ u32 *buffers, memif_if_t *mif, memif_queue_t *mq,
+ memif_per_thread_data_t *ptd, u32 n_left)
{
memif_ring_t *ring = mq->ring;
- u32 *buffers = vlib_frame_vector_args (frame);
- u32 n_left = frame->n_vectors;
u16 slot, free_slots, n_free;
u16 ring_size = 1 << mq->log2_ring_size;
u16 mask = ring_size - 1;
int n_retries = 5;
vlib_buffer_t *b0;
+ u16 head, tail;
retry:
- n_free = ring->tail - mq->last_tail;
+ tail = __atomic_load_n (&ring->tail, __ATOMIC_ACQUIRE);
+ slot = head = ring->head;
+
+ n_free = tail - mq->last_tail;
if (n_free >= 16)
{
vlib_buffer_free_from_ring_no_next (vm, mq->buffers,
mq->last_tail += n_free;
}
- slot = ring->head;
- free_slots = ring_size - ring->head + mq->last_tail;
+ free_slots = ring_size - head + mq->last_tail;
while (n_left && free_slots)
{
memif_desc_t *d0;
u32 bi0;
- CLIB_PREFETCH (&ring->desc[(slot + 8) & mask], CLIB_CACHE_LINE_BYTES,
- STORE);
+ clib_prefetch_store (&ring->desc[(slot + 8) & mask]);
if (PREDICT_TRUE (n_left >= 4))
vlib_prefetch_buffer_header (vlib_get_buffer (vm, buffers[3]), LOAD);
}
no_free_slots:
- CLIB_MEMORY_STORE_BARRIER ();
- ring->head = slot;
+ __atomic_store_n (&ring->head, slot, __ATOMIC_RELEASE);
if (n_left && n_retries--)
goto retry;
- clib_spinlock_unlock_if_init (&mif->lockp);
-
- if (n_left)
- {
- vlib_error_count (vm, node->node_index, MEMIF_TX_ERROR_NO_FREE_SLOTS,
- n_left);
- vlib_buffer_free (vm, buffers, n_left);
- }
-
- if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0 && mq->int_fd > -1)
- {
- u64 b = 1;
- CLIB_UNUSED (int r) = write (mq->int_fd, &b, sizeof (b));
- mq->int_count++;
- }
-
- return frame->n_vectors;
+ return n_left;
}
VNET_DEVICE_CLASS_TX_FN (memif_device_class) (vlib_main_t * vm,
memif_main_t *nm = &memif_main;
vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
memif_if_t *mif = pool_elt_at_index (nm->interfaces, rund->dev_instance);
+ vnet_hw_if_tx_frame_t *tf = vlib_frame_scalar_args (frame);
memif_queue_t *mq;
- u32 thread_index = vm->thread_index;
+ u32 qid = tf->queue_id;
+ u32 *from, thread_index = vm->thread_index;
memif_per_thread_data_t *ptd = vec_elt_at_index (memif_main.per_thread_data,
thread_index);
- u8 tx_queues = vec_len (mif->tx_queues);
+ uword n_left;
- if (tx_queues < vec_len (vlib_mains))
- {
- ASSERT (tx_queues > 0);
- mq = vec_elt_at_index (mif->tx_queues, thread_index % tx_queues);
- clib_spinlock_lock_if_init (&mif->lockp);
- }
- else
- mq = vec_elt_at_index (mif->tx_queues, thread_index);
+ ASSERT (vec_len (mif->tx_queues) > qid);
+ mq = vec_elt_at_index (mif->tx_queues, qid);
+
+ if (tf->shared_queue)
+ clib_spinlock_lock (&mq->lockp);
+ from = vlib_frame_vector_args (frame);
+ n_left = frame->n_vectors;
if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
- return memif_interface_tx_zc_inline (vm, node, frame, mif, mq, ptd);
+ n_left =
+ memif_interface_tx_zc_inline (vm, node, from, mif, mq, ptd, n_left);
else if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
- return memif_interface_tx_inline (vm, node, frame, mif, MEMIF_RING_S2M,
- mq, ptd);
+ n_left = memif_interface_tx_inline (vm, node, from, mif, MEMIF_RING_S2M,
+ mq, ptd, n_left);
else
- return memif_interface_tx_inline (vm, node, frame, mif, MEMIF_RING_M2S,
- mq, ptd);
+ n_left = memif_interface_tx_inline (vm, node, from, mif, MEMIF_RING_M2S,
+ mq, ptd, n_left);
+
+ if (tf->shared_queue)
+ clib_spinlock_unlock (&mq->lockp);
+
+ if (n_left)
+ vlib_error_count (vm, node->node_index, MEMIF_TX_ERROR_NO_FREE_SLOTS,
+ n_left);
+
+ if ((mq->ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0 && mq->int_fd > -1)
+ {
+ u64 b = 1;
+ int __clib_unused r = write (mq->int_fd, &b, sizeof (b));
+ mq->int_count++;
+ }
+
+ if ((mif->flags & MEMIF_IF_FLAG_ZERO_COPY) == 0)
+ vlib_buffer_free (vm, from, frame->n_vectors);
+ else if (n_left)
+ vlib_buffer_free (vm, from + frame->n_vectors - n_left, n_left);
+
+ return frame->n_vectors - n_left;
}
static void
.format_device = format_memif_device,
.format_tx_trace = format_memif_tx_trace,
.tx_function_n_errors = MEMIF_TX_N_ERROR,
- .tx_function_error_strings = memif_tx_func_error_strings,
+ .tx_function_error_counters = memif_tx_func_error_counters,
.rx_redirect_to_node = memif_set_interface_next_node,
.clear_counters = memif_clear_hw_interface_counters,
.admin_up_down_function = memif_interface_admin_up_down,
Code Review / vpp.git / blobdiff