}
static_always_inline uword
-memif_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
- vlib_frame_t * frame, memif_if_t * mif,
- memif_ring_type_t type, memif_queue_t * mq,
- memif_per_thread_data_t * ptd)
+memif_interface_tx_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
+ u32 *buffers, memif_if_t *mif,
+ memif_ring_type_t type, memif_queue_t *mq,
+ memif_per_thread_data_t *ptd, u32 n_left)
{
memif_ring_t *ring;
- u32 *buffers = vlib_frame_vector_args (frame);
- u32 n_left = frame->n_vectors;
u32 n_copy_op;
u16 ring_size, mask, slot, free_slots;
int n_retries = 5;
u32 saved_ptd_buffers_len = _vec_len (ptd->buffers);
u16 saved_slot = slot;
- CLIB_PREFETCH (&ring->desc[(slot + 8) & mask], CLIB_CACHE_LINE_BYTES,
- LOAD);
+ clib_prefetch_load (&ring->desc[(slot + 8) & mask]);
d0 = &ring->desc[slot & mask];
if (PREDICT_FALSE (last_region != d0->region))
{
slot++;
free_slots--;
+ d0->length = dst_off;
d0->flags = MEMIF_DESC_FLAG_NEXT;
d0 = &ring->desc[slot & mask];
dst_off = 0;
co = ptd->copy_ops;
while (n_copy_op >= 8)
{
- CLIB_PREFETCH (co[4].data, CLIB_CACHE_LINE_BYTES, LOAD);
- CLIB_PREFETCH (co[5].data, CLIB_CACHE_LINE_BYTES, LOAD);
- CLIB_PREFETCH (co[6].data, CLIB_CACHE_LINE_BYTES, LOAD);
- CLIB_PREFETCH (co[7].data, CLIB_CACHE_LINE_BYTES, LOAD);
+ clib_prefetch_load (co[4].data);
+ clib_prefetch_load (co[5].data);
+ clib_prefetch_load (co[6].data);
+ clib_prefetch_load (co[7].data);
b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]);
b1 = vlib_get_buffer (vm, ptd->buffers[co[1].buffer_vec_index]);
if (n_left && n_retries--)
goto retry;
- clib_spinlock_unlock_if_init (&mif->lockp);
-
- if (n_left)
- {
- vlib_error_count (vm, node->node_index, MEMIF_TX_ERROR_NO_FREE_SLOTS,
- n_left);
- }
-
- if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0 && mq->int_fd > -1)
- {
- u64 b = 1;
- CLIB_UNUSED (int r) = write (mq->int_fd, &b, sizeof (b));
- mq->int_count++;
- }
-
- vlib_buffer_free (vm, vlib_frame_vector_args (frame), frame->n_vectors);
-
- return frame->n_vectors;
+ return n_left;
}
static_always_inline uword
-memif_interface_tx_zc_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
- vlib_frame_t * frame, memif_if_t * mif,
- memif_queue_t * mq,
- memif_per_thread_data_t * ptd)
+memif_interface_tx_zc_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
+ u32 *buffers, memif_if_t *mif, memif_queue_t *mq,
+ memif_per_thread_data_t *ptd, u32 n_left)
{
memif_ring_t *ring = mq->ring;
- u32 *buffers = vlib_frame_vector_args (frame);
- u32 n_left = frame->n_vectors;
u16 slot, free_slots, n_free;
u16 ring_size = 1 << mq->log2_ring_size;
u16 mask = ring_size - 1;
memif_desc_t *d0;
u32 bi0;
- CLIB_PREFETCH (&ring->desc[(slot + 8) & mask], CLIB_CACHE_LINE_BYTES,
- STORE);
+ clib_prefetch_store (&ring->desc[(slot + 8) & mask]);
if (PREDICT_TRUE (n_left >= 4))
vlib_prefetch_buffer_header (vlib_get_buffer (vm, buffers[3]), LOAD);
if (n_left && n_retries--)
goto retry;
- clib_spinlock_unlock_if_init (&mif->lockp);
-
- if (n_left)
- {
- vlib_error_count (vm, node->node_index, MEMIF_TX_ERROR_NO_FREE_SLOTS,
- n_left);
- vlib_buffer_free (vm, buffers, n_left);
- }
-
- if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0 && mq->int_fd > -1)
- {
- u64 b = 1;
- CLIB_UNUSED (int r) = write (mq->int_fd, &b, sizeof (b));
- mq->int_count++;
- }
-
- return frame->n_vectors;
+ return n_left;
}
VNET_DEVICE_CLASS_TX_FN (memif_device_class) (vlib_main_t * vm,
memif_main_t *nm = &memif_main;
vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
memif_if_t *mif = pool_elt_at_index (nm->interfaces, rund->dev_instance);
+ vnet_hw_if_tx_frame_t *tf = vlib_frame_scalar_args (frame);
memif_queue_t *mq;
- u32 thread_index = vm->thread_index;
+ u32 qid = tf->queue_id;
+ u32 *from, thread_index = vm->thread_index;
memif_per_thread_data_t *ptd = vec_elt_at_index (memif_main.per_thread_data,
thread_index);
- u8 tx_queues = vec_len (mif->tx_queues);
+ uword n_left;
- if (tx_queues < vlib_get_n_threads ())
- {
- ASSERT (tx_queues > 0);
- mq = vec_elt_at_index (mif->tx_queues, thread_index % tx_queues);
- clib_spinlock_lock_if_init (&mif->lockp);
- }
- else
- mq = vec_elt_at_index (mif->tx_queues, thread_index);
+ ASSERT (vec_len (mif->tx_queues) > qid);
+ mq = vec_elt_at_index (mif->tx_queues, qid);
+
+ if (tf->shared_queue)
+ clib_spinlock_lock (&mq->lockp);
+ from = vlib_frame_vector_args (frame);
+ n_left = frame->n_vectors;
if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
- return memif_interface_tx_zc_inline (vm, node, frame, mif, mq, ptd);
+ n_left =
+ memif_interface_tx_zc_inline (vm, node, from, mif, mq, ptd, n_left);
else if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
- return memif_interface_tx_inline (vm, node, frame, mif, MEMIF_RING_S2M,
- mq, ptd);
+ n_left = memif_interface_tx_inline (vm, node, from, mif, MEMIF_RING_S2M,
+ mq, ptd, n_left);
else
- return memif_interface_tx_inline (vm, node, frame, mif, MEMIF_RING_M2S,
- mq, ptd);
+ n_left = memif_interface_tx_inline (vm, node, from, mif, MEMIF_RING_M2S,
+ mq, ptd, n_left);
+
+ if (tf->shared_queue)
+ clib_spinlock_unlock (&mq->lockp);
+
+ if (n_left)
+ vlib_error_count (vm, node->node_index, MEMIF_TX_ERROR_NO_FREE_SLOTS,
+ n_left);
+
+ if ((mq->ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0 && mq->int_fd > -1)
+ {
+ u64 b = 1;
+ int __clib_unused r = write (mq->int_fd, &b, sizeof (b));
+ mq->int_count++;
+ }
+
+ if ((mif->flags & MEMIF_IF_FLAG_ZERO_COPY) == 0)
+ vlib_buffer_free (vm, from, frame->n_vectors);
+ else if (n_left)
+ vlib_buffer_free (vm, from + frame->n_vectors - n_left, n_left);
+
+ return frame->n_vectors - n_left;
}
static void
Code Review / vpp.git / blobdiff