return 0;
}
-/**
- * @brief Spin until the vring is successfully locked
- */
-static_always_inline void
-vhost_user_vring_lock (vhost_user_intf_t * vui, u32 qid)
-{
- clib_spinlock_lock_if_init (&vui->vrings[qid].vring_lock);
-}
-
-/**
- * @brief Unlock the vring lock
- */
-static_always_inline void
-vhost_user_vring_unlock (vhost_user_intf_t * vui, u32 qid)
-{
- clib_spinlock_unlock_if_init (&vui->vrings[qid].vring_lock);
-}
-
static_always_inline void
vhost_user_tx_trace (vhost_trace_t * t,
vhost_user_intf_t * vui, u16 qid,
(!(dst3 = map_guest_mem (vui, cpy[3].dst, map_hint))))
return 1;
- CLIB_PREFETCH ((void *) cpy[2].src, 64, LOAD);
- CLIB_PREFETCH ((void *) cpy[3].src, 64, LOAD);
+ clib_prefetch_load ((void *) cpy[2].src);
+ clib_prefetch_load ((void *) cpy[3].src);
clib_memcpy_fast (dst0, (void *) cpy[0].src, cpy[0].len);
clib_memcpy_fast (dst1, (void *) cpy[1].src, cpy[1].len);
}
static_always_inline uword
-vhost_user_device_class_packed (vlib_main_t * vm, vlib_node_runtime_t * node,
- vlib_frame_t * frame)
+vhost_user_device_class_packed (vlib_main_t *vm, vlib_node_runtime_t *node,
+ vlib_frame_t *frame, vhost_user_intf_t *vui,
+ vhost_user_vring_t *rxvq)
{
u32 *buffers = vlib_frame_vector_args (frame);
u32 n_left = frame->n_vectors;
vhost_user_main_t *vum = &vhost_user_main;
- vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
- vhost_user_intf_t *vui =
- pool_elt_at_index (vum->vhost_user_interfaces, rd->dev_instance);
- u32 qid;
- vhost_user_vring_t *rxvq;
+ u32 qid = rxvq->qid;
u8 error;
u32 thread_index = vm->thread_index;
vhost_cpu_t *cpu = &vum->cpus[thread_index];
u16 n_descs_processed;
u8 indirect, chained;
- qid = VHOST_VRING_IDX_RX (*vec_elt_at_index (vui->per_cpu_tx_qid,
- thread_index));
- rxvq = &vui->vrings[qid];
-
retry:
error = VHOST_USER_TX_FUNC_ERROR_NONE;
tx_headers_len = 0;
buffer_map_addr += cpy->len;
desc_len += cpy->len;
- CLIB_PREFETCH (&rxvq->packed_desc, CLIB_CACHE_LINE_BYTES, LOAD);
+ clib_prefetch_load (&rxvq->packed_desc);
/* Check if vlib buffer has more data. If not, get more or break */
if (PREDICT_TRUE (!bytes_left))
goto retry;
}
- vhost_user_vring_unlock (vui, qid);
+ clib_spinlock_unlock (&rxvq->vring_lock);
if (PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE))
{
vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
vhost_user_intf_t *vui =
pool_elt_at_index (vum->vhost_user_interfaces, rd->dev_instance);
- u32 qid = ~0;
+ u32 qid;
vhost_user_vring_t *rxvq;
u8 error;
u32 thread_index = vm->thread_index;
u16 copy_len;
u16 tx_headers_len;
u32 or_flags;
+ vnet_hw_if_tx_frame_t *tf = vlib_frame_scalar_args (frame);
if (PREDICT_FALSE (!vui->admin_up))
{
goto done3;
}
- qid = VHOST_VRING_IDX_RX (*vec_elt_at_index (vui->per_cpu_tx_qid,
- thread_index));
+ qid = VHOST_VRING_IDX_RX (tf->queue_id);
rxvq = &vui->vrings[qid];
+ ASSERT (tf->queue_id == rxvq->qid);
+
if (PREDICT_FALSE (rxvq->avail == 0))
{
error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
goto done3;
}
-
- if (PREDICT_FALSE (vui->use_tx_spinlock))
- vhost_user_vring_lock (vui, qid);
+ if (tf->shared_queue)
+ clib_spinlock_lock (&rxvq->vring_lock);
if (vhost_user_is_packed_ring_supported (vui))
- return (vhost_user_device_class_packed (vm, node, frame));
+ return (vhost_user_device_class_packed (vm, node, frame, vui, rxvq));
retry:
error = VHOST_USER_TX_FUNC_ERROR_NONE;
buffer_map_addr += cpy->len;
desc_len += cpy->len;
- CLIB_PREFETCH (&rxvq->desc, CLIB_CACHE_LINE_BYTES, LOAD);
+ clib_prefetch_load (&rxvq->desc);
}
// Check if vlib buffer has more data. If not, get more or break.
vhost_user_send_call (vm, vui, rxvq);
}
- vhost_user_vring_unlock (vui, qid);
+ clib_spinlock_unlock (&rxvq->vring_lock);
done3:
if (PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE))
vhost_user_intf_t *vui =
pool_elt_at_index (vum->vhost_user_interfaces, hif->dev_instance);
vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
+ vhost_cpu_t *cpu;
+ if (mode == txvq->mode)
+ return 0;
+
+ if ((mode != VNET_HW_IF_RX_MODE_POLLING) &&
+ (mode != VNET_HW_IF_RX_MODE_ADAPTIVE) &&
+ (mode != VNET_HW_IF_RX_MODE_INTERRUPT))
+ {
+ vu_log_err (vui, "unhandled mode %d changed for if %d queue %d", mode,
+ hw_if_index, qid);
+ return clib_error_return (0, "unsupported");
+ }
+
+ if (txvq->thread_index == ~0)
+ return clib_error_return (0, "Queue initialization is not finished yet");
+
+ cpu = vec_elt_at_index (vum->cpus, txvq->thread_index);
if ((mode == VNET_HW_IF_RX_MODE_INTERRUPT) ||
(mode == VNET_HW_IF_RX_MODE_ADAPTIVE))
{
}
if (txvq->mode == VNET_HW_IF_RX_MODE_POLLING)
{
+ ASSERT (cpu->polling_q_count != 0);
+ if (cpu->polling_q_count)
+ cpu->polling_q_count--;
vum->ifq_count++;
// Start the timer if this is the first encounter on interrupt
// interface/queue
if ((vum->ifq_count == 1) &&
- (vum->coalesce_time > 0.0) && (vum->coalesce_frames > 0))
+ ((vum->coalesce_time > 0.0) || (vum->coalesce_frames > 0)))
vlib_process_signal_event (vm,
vhost_user_send_interrupt_node.index,
VHOST_USER_EVENT_START_TIMER, 0);
if (((txvq->mode == VNET_HW_IF_RX_MODE_INTERRUPT) ||
(txvq->mode == VNET_HW_IF_RX_MODE_ADAPTIVE)) && vum->ifq_count)
{
+ cpu->polling_q_count++;
vum->ifq_count--;
// Stop the timer if there is no more interrupt interface/queue
- if ((vum->ifq_count == 0) &&
- (vum->coalesce_time > 0.0) && (vum->coalesce_frames > 0))
+ if (vum->ifq_count == 0)
vlib_process_signal_event (vm,
vhost_user_send_interrupt_node.index,
VHOST_USER_EVENT_STOP_TIMER, 0);
}
txvq->mode = mode;
- if (mode == VNET_HW_IF_RX_MODE_POLLING)
- txvq->used->flags = VRING_USED_F_NO_NOTIFY;
- else if ((mode == VNET_HW_IF_RX_MODE_ADAPTIVE) ||
- (mode == VNET_HW_IF_RX_MODE_INTERRUPT))
- txvq->used->flags = 0;
- else
- {
- vu_log_err (vui, "unhandled mode %d changed for if %d queue %d", mode,
- hw_if_index, qid);
- return clib_error_return (0, "unsupported");
- }
+ vhost_user_set_operation_mode (vui, txvq);
return 0;
}