#include <vlib/vlib.h>
#include <vlib/unix/unix.h>
-#include <vnet/ip/ip.h>
-
#include <vnet/ethernet/ethernet.h>
#include <vnet/devices/devices.h>
#include <vnet/feature/feature.h>
return 0;
}
-/**
- * @brief Try once to lock the vring
- * @return 0 on success, non-zero on failure.
- */
-static_always_inline int
-vhost_user_vring_try_lock (vhost_user_intf_t * vui, u32 qid)
-{
- return clib_atomic_test_and_set (vui->vring_locks[qid]);
-}
-
/**
* @brief Spin until the vring is successfully locked
*/
static_always_inline void
vhost_user_vring_lock (vhost_user_intf_t * vui, u32 qid)
{
- while (vhost_user_vring_try_lock (vui, qid))
- ;
+ clib_spinlock_lock_if_init (&vui->vrings[qid].vring_lock);
}
/**
static_always_inline void
vhost_user_vring_unlock (vhost_user_intf_t * vui, u32 qid)
{
- clib_atomic_release (vui->vring_locks[qid]);
+ clib_spinlock_unlock_if_init (&vui->vrings[qid].vring_lock);
}
static_always_inline void
(!(dst3 = map_guest_mem (vui, cpy[3].dst, map_hint))))
return 1;
- CLIB_PREFETCH ((void *) cpy[2].src, 64, LOAD);
- CLIB_PREFETCH ((void *) cpy[3].src, 64, LOAD);
+ clib_prefetch_load ((void *) cpy[2].src);
+ clib_prefetch_load ((void *) cpy[3].src);
clib_memcpy_fast (dst0, (void *) cpy[0].src, cpy[0].len);
clib_memcpy_fast (dst1, (void *) cpy[1].src, cpy[1].len);
generic_header_offset_t gho = { 0 };
int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
+ vnet_buffer_oflags_t oflags = vnet_buffer (b)->oflags;
ASSERT (!(is_ip4 && is_ip6));
vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6);
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
{
ip4_header_t *ip4;
}
/* checksum offload */
- if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset;
hdr->csum_offset = offsetof (udp_header_t, checksum);
- udp_header_t *udp =
- (udp_header_t *) (vlib_buffer_get_current (b) + gho.l4_hdr_offset);
- udp->checksum = 0;
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset;
hdr->csum_offset = offsetof (tcp_header_t, checksum);
- tcp_header_t *tcp =
- (tcp_header_t *) (vlib_buffer_get_current (b) + gho.l4_hdr_offset);
- tcp->checksum = 0;
}
/* GSO offload */
if (b->flags & VNET_BUFFER_F_GSO)
{
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
if (is_ip4 &&
(vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)))
}
}
else if ((vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)) &&
- (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
{
hdr->gso_size = vnet_buffer2 (b)->gso_size;
hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
}
static_always_inline void
-vhost_user_mark_desc_available (vlib_main_t * vm, vhost_user_vring_t * rxvq,
+vhost_user_mark_desc_available (vlib_main_t * vm, vhost_user_intf_t * vui,
+ vhost_user_vring_t * rxvq,
u16 * n_descs_processed, u8 chained,
vlib_frame_t * frame, u32 n_left)
{
rxvq->n_since_last_int += frame->n_vectors - n_left;
if (rxvq->n_since_last_int > vum->coalesce_frames)
- vhost_user_send_call (vm, rxvq);
+ vhost_user_send_call (vm, vui, rxvq);
}
}
hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
hdr->num_buffers = 1;
- or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+ or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
/* Guest supports csum offload and buffer requires checksum offload? */
if (or_flags &&
buffer_map_addr += cpy->len;
desc_len += cpy->len;
- CLIB_PREFETCH (&rxvq->packed_desc, CLIB_CACHE_LINE_BYTES, LOAD);
+ clib_prefetch_load (&rxvq->packed_desc);
/* Check if vlib buffer has more data. If not, get more or break */
if (PREDICT_TRUE (!bytes_left))
copy_len = 0;
/* give buffers back to driver */
- vhost_user_mark_desc_available (vm, rxvq, &n_descs_processed,
+ vhost_user_mark_desc_available (vm, vui, rxvq, &n_descs_processed,
chained, frame, n_left);
}
vlib_error_count (vm, node->node_index,
VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- vhost_user_mark_desc_available (vm, rxvq, &n_descs_processed, chained,
- frame, n_left);
+ vhost_user_mark_desc_available (vm, vui, rxvq, &n_descs_processed,
+ chained, frame, n_left);
}
/*
hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
hdr->num_buffers = 1; //This is local, no need to check
- or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+ or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
/* Guest supports csum offload and buffer requires checksum offload? */
if (or_flags
buffer_map_addr += cpy->len;
desc_len += cpy->len;
- CLIB_PREFETCH (&rxvq->desc, CLIB_CACHE_LINE_BYTES, LOAD);
+ clib_prefetch_load (&rxvq->desc);
}
// Check if vlib buffer has more data. If not, get more or break.
rxvq->n_since_last_int += frame->n_vectors - n_left;
if (rxvq->n_since_last_int > vum->coalesce_frames)
- vhost_user_send_call (vm, rxvq);
+ vhost_user_send_call (vm, vui, rxvq);
}
vhost_user_vring_unlock (vui, qid);
static __clib_unused clib_error_t *
vhost_user_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index,
- u32 qid, vnet_hw_interface_rx_mode mode)
+ u32 qid, vnet_hw_if_rx_mode mode)
{
vlib_main_t *vm = vnm->vlib_main;
vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
vhost_user_intf_t *vui =
pool_elt_at_index (vum->vhost_user_interfaces, hif->dev_instance);
vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
+ vhost_cpu_t *cpu;
+
+ if (mode == txvq->mode)
+ return 0;
+
+ if ((mode != VNET_HW_IF_RX_MODE_POLLING) &&
+ (mode != VNET_HW_IF_RX_MODE_ADAPTIVE) &&
+ (mode != VNET_HW_IF_RX_MODE_INTERRUPT))
+ {
+ vu_log_err (vui, "unhandled mode %d changed for if %d queue %d", mode,
+ hw_if_index, qid);
+ return clib_error_return (0, "unsupported");
+ }
- if ((mode == VNET_HW_INTERFACE_RX_MODE_INTERRUPT) ||
- (mode == VNET_HW_INTERFACE_RX_MODE_ADAPTIVE))
+ if (txvq->thread_index == ~0)
+ return clib_error_return (0, "Queue initialization is not finished yet");
+
+ cpu = vec_elt_at_index (vum->cpus, txvq->thread_index);
+ if ((mode == VNET_HW_IF_RX_MODE_INTERRUPT) ||
+ (mode == VNET_HW_IF_RX_MODE_ADAPTIVE))
{
if (txvq->kickfd_idx == ~0)
{
// We cannot support interrupt mode if the driver opts out
return clib_error_return (0, "Driver does not support interrupt");
}
- if (txvq->mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
+ if (txvq->mode == VNET_HW_IF_RX_MODE_POLLING)
{
+ ASSERT (cpu->polling_q_count != 0);
+ if (cpu->polling_q_count)
+ cpu->polling_q_count--;
vum->ifq_count++;
// Start the timer if this is the first encounter on interrupt
// interface/queue
if ((vum->ifq_count == 1) &&
- (vum->coalesce_time > 0.0) && (vum->coalesce_frames > 0))
+ ((vum->coalesce_time > 0.0) || (vum->coalesce_frames > 0)))
vlib_process_signal_event (vm,
vhost_user_send_interrupt_node.index,
VHOST_USER_EVENT_START_TIMER, 0);
}
}
- else if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
+ else if (mode == VNET_HW_IF_RX_MODE_POLLING)
{
- if (((txvq->mode == VNET_HW_INTERFACE_RX_MODE_INTERRUPT) ||
- (txvq->mode == VNET_HW_INTERFACE_RX_MODE_ADAPTIVE)) &&
- vum->ifq_count)
+ if (((txvq->mode == VNET_HW_IF_RX_MODE_INTERRUPT) ||
+ (txvq->mode == VNET_HW_IF_RX_MODE_ADAPTIVE)) && vum->ifq_count)
{
+ cpu->polling_q_count++;
vum->ifq_count--;
// Stop the timer if there is no more interrupt interface/queue
- if ((vum->ifq_count == 0) &&
- (vum->coalesce_time > 0.0) && (vum->coalesce_frames > 0))
+ if (vum->ifq_count == 0)
vlib_process_signal_event (vm,
vhost_user_send_interrupt_node.index,
VHOST_USER_EVENT_STOP_TIMER, 0);
}
txvq->mode = mode;
- if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
- txvq->used->flags = VRING_USED_F_NO_NOTIFY;
- else if ((mode == VNET_HW_INTERFACE_RX_MODE_ADAPTIVE) ||
- (mode == VNET_HW_INTERFACE_RX_MODE_INTERRUPT))
- txvq->used->flags = 0;
- else
- {
- vu_log_err (vui, "unhandled mode %d changed for if %d queue %d", mode,
- hw_if_index, qid);
- return clib_error_return (0, "unsupported");
- }
+ vhost_user_set_operation_mode (vui, txvq);
return 0;
}
Code Review / vpp.git / blobdiff