_(NONE, "no error") \
_(NOT_READY, "vhost user state error") \
_(PKT_DROP_NOBUF, "tx packet drops (no available descriptors)") \
- _(MMAP_FAIL, "mmap failure")
+ _(PKT_DROP_NOMRG, "tx packet drops (cannot merge descriptors)") \
+ _(MMAP_FAIL, "mmap failure") \
+ _(INDIRECT_OVERFLOW, "indirect descriptor table overflow")
typedef enum
{
_(NO_ERROR, "no error") \
_(NO_BUFFER, "no available buffer") \
_(MMAP_FAIL, "mmap failure") \
- _(UNDERSIZED_FRAME, "undersized ethernet frame received (< 14 bytes)")
+ _(INDIRECT_OVERFLOW, "indirect descriptor overflows table") \
+ _(UNDERSIZED_FRAME, "undersized ethernet frame received (< 14 bytes)") \
+ _(FULL_RX_QUEUE, "full rx queue (possible driver tx drop)")
typedef enum
{
map_guest_mem (vhost_user_intf_t * vui, uword addr)
{
int i;
+#if __SSE4_2__
+ __m128i rl, rh, al, ah, r;
+ al = _mm_set1_epi64x (addr + 1);
+ ah = _mm_set1_epi64x (addr);
+
+ rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[0]);
+ rl = _mm_cmpgt_epi64 (al, rl);
+ rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[0]);
+ rh = _mm_cmpgt_epi64 (rh, ah);
+ r = _mm_and_si128 (rl, rh);
+
+ rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[2]);
+ rl = _mm_cmpgt_epi64 (al, rl);
+ rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[2]);
+ rh = _mm_cmpgt_epi64 (rh, ah);
+ r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x22);
+
+ rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[4]);
+ rl = _mm_cmpgt_epi64 (al, rl);
+ rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[4]);
+ rh = _mm_cmpgt_epi64 (rh, ah);
+ r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x44);
+
+ rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[6]);
+ rl = _mm_cmpgt_epi64 (al, rl);
+ rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[6]);
+ rh = _mm_cmpgt_epi64 (rh, ah);
+ r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x88);
+
+ r = _mm_shuffle_epi8 (r, _mm_set_epi64x (0, 0x0e060c040a020800));
+ i = __builtin_ctzll (_mm_movemask_epi8 (r));
+
+ if (i < vui->nregions)
+ {
+ return (void *) (vui->region_mmap_addr[i] + addr -
+ vui->regions[i].guest_phys_addr);
+ }
+
+#else
for (i = 0; i < vui->nregions; i++)
{
if ((vui->regions[i].guest_phys_addr <= addr) &&
vui->regions[i].guest_phys_addr);
}
}
+#endif
DBG_VQ ("failed to map guest mem addr %llx", addr);
return 0;
}
unix_file_del (&unix_main, unix_main.file_pool + vui->unix_file_index);
vui->unix_file_index = ~0;
}
+ else
+ close (vui->unix_fd);
hash_unset (vum->vhost_user_interface_index_by_sock_fd, vui->unix_fd);
hash_unset (vum->vhost_user_interface_index_by_listener_fd, vui->unix_fd);
- close (vui->unix_fd);
vui->unix_fd = -1;
vui->is_up = 0;
for (q = 0; q < vui->num_vrings; q++)
{
+ if (vui->vrings[q].callfd > -1)
+ {
+ unix_file_t *uf = pool_elt_at_index (unix_main.file_pool,
+ vui->vrings[q].callfd_idx);
+ unix_file_del (&unix_main, uf);
+ }
+
+ if (vui->vrings[q].kickfd > -1)
+ close (vui->vrings[q].kickfd);
+
+ vui->vrings[q].callfd = -1;
+ vui->vrings[q].kickfd = -1;
vui->vrings[q].desc = NULL;
vui->vrings[q].avail = NULL;
vui->vrings[q].used = NULL;
msg.flags |= 4;
msg.u64 = (1 << FEAT_VIRTIO_NET_F_MRG_RXBUF) |
(1 << FEAT_VIRTIO_F_ANY_LAYOUT) |
+ (1 << FEAT_VIRTIO_F_INDIRECT_DESC) |
(1 << FEAT_VHOST_F_LOG_ALL) |
(1 << FEAT_VIRTIO_NET_F_GUEST_ANNOUNCE) |
- (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES);
+ (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES) |
+ (1UL << FEAT_VIRTIO_F_VERSION_1);
msg.u64 &= vui->feature_mask;
msg.size = sizeof (msg.u64);
vui->region_mmap_addr[i] = mmap (0, map_sz, PROT_READ | PROT_WRITE,
MAP_SHARED, fds[i], 0);
+ vui->region_guest_addr_lo[i] = vui->regions[i].guest_phys_addr;
+ vui->region_guest_addr_hi[i] = vui->regions[i].guest_phys_addr +
+ vui->regions[i].memory_size;
DBG_SOCK
("map memory region %d addr 0 len 0x%lx fd %d mapped 0x%lx "
goto close_socket;
/* if there is old fd, delete it */
- if (vui->vrings[q].callfd)
+ if (vui->vrings[q].callfd > -1)
{
unix_file_t *uf = pool_elt_at_index (unix_main.file_pool,
vui->vrings[q].callfd_idx);
if (number_of_fds != 1)
goto close_socket;
+ if (vui->vrings[q].kickfd > -1)
+ close (vui->vrings[q].kickfd);
+
vui->vrings[q].kickfd = fds[0];
}
else
clib_error_t *error;
vhost_user_main_t *vum = &vhost_user_main;
vlib_thread_main_t *tm = vlib_get_thread_main ();
+ vlib_thread_registration_t *tr;
+ uword *p;
error = vlib_call_init_function (vm, ip4_init);
if (error)
vec_validate_aligned (vum->rx_buffers, tm->n_vlib_mains - 1,
CLIB_CACHE_LINE_BYTES);
+ /* find out which cpus will be used for input */
+ vum->input_cpu_first_index = 0;
+ vum->input_cpu_count = 1;
+ p = hash_get_mem (tm->thread_registrations_by_name, "workers");
+ tr = p ? (vlib_thread_registration_t *) p[0] : 0;
+
+ if (tr && tr->count > 0)
+ {
+ vum->input_cpu_first_index = tr->first_index;
+ vum->input_cpu_count = tr->count;
+ }
+
return 0;
}
if (PREDICT_FALSE (txvq->avail->flags & 0xFFFE))
return 0;
+ n_left = (u16) (txvq->avail->idx - txvq->last_avail_idx);
+
/* nothing to do */
- if (txvq->avail->idx == txvq->last_avail_idx)
+ if (PREDICT_FALSE (n_left == 0))
return 0;
- if (PREDICT_TRUE (txvq->avail->idx > txvq->last_avail_idx))
- n_left = txvq->avail->idx - txvq->last_avail_idx;
- else /* wrapped */
- n_left = (u16) - 1 - txvq->last_avail_idx + txvq->avail->idx;
+ if (PREDICT_FALSE (n_left == txvq->qsz))
+ {
+ //Informational error logging when VPP is not receiving packets fast enough
+ vlib_error_count (vm, node->node_index,
+ VHOST_USER_INPUT_FUNC_ERROR_FULL_RX_QUEUE, 1);
+ }
if (PREDICT_FALSE (!vui->admin_up))
{
return 0;
}
- if (PREDICT_FALSE (n_left > txvq->qsz))
- return 0;
-
qsz_mask = txvq->qsz - 1;
cpu_index = os_get_cpu_number ();
drops = 0;
*/
if (PREDICT_FALSE (!vum->rx_buffers[cpu_index]))
{
- vec_alloc (vum->rx_buffers[cpu_index], VLIB_FRAME_SIZE);
+ vec_alloc (vum->rx_buffers[cpu_index], 2 * VLIB_FRAME_SIZE);
if (PREDICT_FALSE (!vum->rx_buffers[cpu_index]))
flush = n_left; //Drop all input
if (PREDICT_FALSE (_vec_len (vum->rx_buffers[cpu_index]) < n_left))
{
+ u32 curr_len = _vec_len (vum->rx_buffers[cpu_index]);
_vec_len (vum->rx_buffers[cpu_index]) +=
vlib_buffer_alloc_from_free_list (vm,
vum->rx_buffers[cpu_index] +
- _vec_len (vum->rx_buffers
- [cpu_index]),
- VLIB_FRAME_SIZE -
- _vec_len (vum->rx_buffers
- [cpu_index]),
+ curr_len,
+ 2 * VLIB_FRAME_SIZE - curr_len,
VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
if (PREDICT_FALSE (n_left > _vec_len (vum->rx_buffers[cpu_index])))
u16 desc_chain_head, desc_current;
u8 error = VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR;
+ if (PREDICT_TRUE (n_left > 1))
+ {
+ u32 next_desc =
+ txvq->avail->ring[(txvq->last_avail_idx + 1) & qsz_mask];
+ void *buffer_addr =
+ map_guest_mem (vui, txvq->desc[next_desc].addr);
+ if (PREDICT_TRUE (buffer_addr != 0))
+ CLIB_PREFETCH (buffer_addr, 64, STORE);
+
+ u32 bi = vum->rx_buffers[cpu_index][rx_len - 2];
+ vlib_prefetch_buffer_with_index (vm, bi, STORE);
+ CLIB_PREFETCH (vlib_get_buffer (vm, bi)->data, 128, STORE);
+ }
+
desc_chain_head = desc_current =
txvq->avail->ring[txvq->last_avail_idx & qsz_mask];
bi_head = bi_current = vum->rx_buffers[cpu_index][--rx_len];
uword offset;
if (PREDICT_TRUE (vui->is_any_layout) ||
- !(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT))
+ (!(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) &&
+ !(txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)))
{
/* ANYLAYOUT or single buffer */
offset = vui->virtio_net_hdr_sz;
offset = txvq->desc[desc_current].len;
}
+ vring_desc_t *desc_table = txvq->desc;
+ u32 desc_index = desc_current;
+
+ if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
+ {
+ desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr);
+ desc_index = 0;
+ if (PREDICT_FALSE (desc_table == 0))
+ {
+ error = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
+ goto out;
+ }
+ }
+
while (1)
{
void *buffer_addr =
- map_guest_mem (vui, txvq->desc[desc_current].addr);
+ map_guest_mem (vui, desc_table[desc_index].addr);
if (PREDICT_FALSE (buffer_addr == 0))
{
error = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
- break;
+ goto out;
+ }
+
+ if (PREDICT_TRUE
+ (desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT))
+ {
+ CLIB_PREFETCH (&desc_table[desc_table[desc_index].next],
+ sizeof (vring_desc_t), STORE);
}
#if VHOST_USER_COPY_TX_HDR == 1
clib_memcpy (b->pre_data, buffer_addr, sizeof (virtio_net_hdr_t)); /* 12 byte hdr is not used on tx */
#endif
- if (txvq->desc[desc_current].len > offset)
+ if (desc_table[desc_index].len > offset)
{
- u16 len = txvq->desc[desc_current].len - offset;
+ u16 len = desc_table[desc_index].len - offset;
u16 copied = vlib_buffer_chain_append_data_with_alloc (vm,
VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX,
b_head,
+
offset,
len);
-
if (copied != len)
{
error = VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER;
offset = 0;
/* if next flag is set, take next desc in the chain */
- if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT)
- desc_current = txvq->desc[desc_current].next;
+ if ((desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT))
+ desc_index = desc_table[desc_index].next;
else
- break;
+ goto out;
}
+ out:
/* consume the descriptor and return it as used */
txvq->last_avail_idx++;
ring[txvq->last_used_idx & qsz_mask]);
txvq->last_used_idx++;
+ //It is important to free RX as fast as possible such that the TX
+ //process does not drop packets
+ if ((txvq->last_used_idx & 0x3f) == 0) // Every 64 packets
+ txvq->used->idx = txvq->last_used_idx;
+
if (PREDICT_FALSE (b_head->current_length < 14 &&
error == VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR))
- {
- error = VHOST_USER_INPUT_FUNC_ERROR_UNDERSIZED_FRAME;
- }
+ error = VHOST_USER_INPUT_FUNC_ERROR_UNDERSIZED_FRAME;
VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b_head);
}
vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+
}
if (PREDICT_TRUE (vum->rx_buffers[cpu_index] != 0))
}
/* interrupt (call) handling */
- if ((txvq->callfd > 0) && !(txvq->avail->flags & 1))
+ if ((txvq->callfd > -1) && !(txvq->avail->flags & 1))
{
txvq->n_since_last_int += n_rx_packets;
vlib_node_runtime_t * node, vlib_frame_t * f)
{
vhost_user_main_t *vum = &vhost_user_main;
-#if DPDK > 0
- dpdk_main_t *dm = &dpdk_main;
u32 cpu_index = os_get_cpu_number ();
-#endif
vhost_user_intf_t *vui;
uword n_rx_packets = 0;
int i;
vui = vec_elt_at_index (vum->vhost_user_interfaces, i);
if (vui->is_up)
{
-#if DPDK > 0
- if ((i % dm->input_cpu_count) ==
- (cpu_index - dm->input_cpu_first_index))
-#endif
+ if ((i % vum->input_cpu_count) ==
+ (cpu_index - vum->input_cpu_first_index))
n_rx_packets += vhost_user_if_input (vm, vum, vui, node);
}
}
{
u32 *buffers = vlib_frame_args (frame);
u32 n_left = 0;
- u16 used_index;
vhost_user_main_t *vum = &vhost_user_main;
uword n_packets = 0;
vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
u16 qsz_mask;
u8 error = VHOST_USER_TX_FUNC_ERROR_NONE;
+ n_left = n_packets = frame->n_vectors;
+
if (PREDICT_FALSE (!vui->is_up))
goto done2;
goto done2;
}
- n_left = n_packets = frame->n_vectors;
- used_index = rxvq->used->idx;
qsz_mask = rxvq->qsz - 1; /* qsz is always power of 2 */
while (n_left > 0)
{
vlib_buffer_t *b0, *current_b0;
- u16 desc_chain_head, desc_current, desc_len;
+ u16 desc_head, desc_index, desc_len;
+ vring_desc_t *desc_table;
void *buffer_addr;
- uword offset;
-
- if (n_left >= 2)
- vlib_prefetch_buffer_with_index (vm, buffers[1], LOAD);
+ u32 buffer_len;
b0 = vlib_get_buffer (vm, buffers[0]);
buffers++;
- n_left--;
if (PREDICT_FALSE (rxvq->last_avail_idx == rxvq->avail->idx))
{
goto done;
}
- desc_current = desc_chain_head =
+ desc_table = rxvq->desc;
+ desc_head = desc_index =
rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
- offset = vui->virtio_net_hdr_sz;
- desc_len = offset;
+ if (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT)
+ {
+ if (PREDICT_FALSE
+ (rxvq->desc[desc_head].len < sizeof (vring_desc_t)))
+ {
+ error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
+ goto done;
+ }
+ if (PREDICT_FALSE
+ (!(desc_table =
+ map_guest_mem (vui, rxvq->desc[desc_index].addr))))
+ {
+ error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
+ goto done;
+ }
+ desc_index = 0;
+ }
+
+ desc_len = vui->virtio_net_hdr_sz;
+
if (PREDICT_FALSE
- (!(buffer_addr =
- map_guest_mem (vui, rxvq->desc[desc_current].addr))))
+ (!(buffer_addr = map_guest_mem (vui, desc_table[desc_index].addr))))
{
error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
goto done;
}
- CLIB_PREFETCH (buffer_addr, clib_min (rxvq->desc[desc_current].len,
- 4 * CLIB_CACHE_LINE_BYTES),
- STORE);
+ buffer_len = desc_table[desc_index].len;
+
+ CLIB_PREFETCH (buffer_addr,
+ clib_min (buffer_len, 2 * CLIB_CACHE_LINE_BYTES), STORE);
virtio_net_hdr_mrg_rxbuf_t *hdr =
(virtio_net_hdr_mrg_rxbuf_t *) buffer_addr;
hdr->hdr.flags = 0;
hdr->hdr.gso_type = 0;
-
- vhost_user_log_dirty_pages (vui, rxvq->desc[desc_current].addr,
- vui->virtio_net_hdr_sz);
-
if (vui->virtio_net_hdr_sz == 12)
hdr->num_buffers = 1;
+ vhost_user_log_dirty_pages (vui, desc_table[desc_index].addr,
+ vui->virtio_net_hdr_sz);
+
u16 bytes_left = b0->current_length;
- buffer_addr += offset;
+ buffer_addr += vui->virtio_net_hdr_sz;
+ buffer_len -= vui->virtio_net_hdr_sz;
current_b0 = b0;
-
- //FIXME: This was in the code but I don't think it is valid
- /*if (PREDICT_FALSE(!vui->is_any_layout && (rxvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT))) {
- rxvq->desc[desc_current].len = vui->virtio_net_hdr_sz;
- } */
-
while (1)
{
if (!bytes_left)
}
}
- if (rxvq->desc[desc_current].len <= offset)
+ if (buffer_len == 0)
{ //Get new output
- if (rxvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT)
+ if (desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT)
{
- offset = 0;
- desc_current = rxvq->desc[desc_current].next;
+ //Next one is chained
+ desc_index = desc_table[desc_index].next;
if (PREDICT_FALSE
(!(buffer_addr =
- map_guest_mem (vui, rxvq->desc[desc_current].addr))))
+ map_guest_mem (vui, desc_table[desc_index].addr))))
{
- used_index -= hdr->num_buffers - 1;
+ rxvq->last_used_idx -= hdr->num_buffers - 1;
rxvq->last_avail_idx -= hdr->num_buffers - 1;
error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
goto done;
}
+ buffer_len = desc_table[desc_index].len;
}
- else if (vui->virtio_net_hdr_sz == 12)
- { //MRG is available
-
+ else if (vui->virtio_net_hdr_sz == 12) //MRG is available
+ {
//Move from available to used buffer
- rxvq->used->ring[used_index & qsz_mask].id =
- desc_chain_head;
- rxvq->used->ring[used_index & qsz_mask].len = desc_len;
+ rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id =
+ desc_head;
+ rxvq->used->ring[rxvq->last_used_idx & qsz_mask].len =
+ desc_len;
vhost_user_log_dirty_ring (vui, rxvq,
- ring[used_index & qsz_mask]);
+ ring[rxvq->last_used_idx &
+ qsz_mask]);
rxvq->last_avail_idx++;
- used_index++;
+ rxvq->last_used_idx++;
hdr->num_buffers++;
if (PREDICT_FALSE
(rxvq->last_avail_idx == rxvq->avail->idx))
{
//Dequeue queued descriptors for this packet
- used_index -= hdr->num_buffers - 1;
+ rxvq->last_used_idx -= hdr->num_buffers - 1;
rxvq->last_avail_idx -= hdr->num_buffers - 1;
error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
goto done;
}
- //Look at next one
- desc_chain_head =
+ desc_table = rxvq->desc;
+ desc_head = desc_index =
rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
- desc_current = desc_chain_head;
- desc_len = 0;
- offset = 0;
+ if (PREDICT_FALSE
+ (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT))
+ {
+ //It is seriously unlikely that a driver will put indirect descriptor
+ //after non-indirect descriptor.
+ if (PREDICT_FALSE
+ (rxvq->desc[desc_head].len < sizeof (vring_desc_t)))
+ {
+ error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
+ goto done;
+ }
+ if (PREDICT_FALSE
+ (!(desc_table =
+ map_guest_mem (vui,
+ rxvq->desc[desc_index].addr))))
+ {
+ error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
+ goto done;
+ }
+ desc_index = 0;
+ }
+
if (PREDICT_FALSE
(!(buffer_addr =
- map_guest_mem (vui, rxvq->desc[desc_current].addr))))
+ map_guest_mem (vui, desc_table[desc_index].addr))))
{
- //Dequeue queued descriptors for this packet
- used_index -= hdr->num_buffers - 1;
- rxvq->last_avail_idx -= hdr->num_buffers - 1;
error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
goto done;
}
+ buffer_len = desc_table[desc_index].len;
+ CLIB_PREFETCH (buffer_addr,
+ clib_min (buffer_len,
+ 2 * CLIB_CACHE_LINE_BYTES), STORE);
}
else
{
- error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
+ error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOMRG;
goto done;
}
}
- u16 bytes_to_copy =
- bytes_left >
- (rxvq->desc[desc_current].len -
- offset) ? (rxvq->desc[desc_current].len - offset) : bytes_left;
+ u16 bytes_to_copy = bytes_left;
+ bytes_to_copy =
+ (bytes_to_copy > buffer_len) ? buffer_len : bytes_to_copy;
clib_memcpy (buffer_addr,
vlib_buffer_get_current (current_b0) +
current_b0->current_length - bytes_left,
bytes_to_copy);
vhost_user_log_dirty_pages (vui,
- rxvq->desc[desc_current].addr + offset,
+ desc_table[desc_index].addr +
+ desc_table[desc_index].len -
+ bytes_left - bytes_to_copy,
bytes_to_copy);
+
bytes_left -= bytes_to_copy;
- offset += bytes_to_copy;
+ buffer_len -= bytes_to_copy;
buffer_addr += bytes_to_copy;
desc_len += bytes_to_copy;
}
+ if (PREDICT_TRUE (n_left >= 2))
+ {
+ vlib_prefetch_buffer_with_index (vm, buffers[1], STORE);
+ CLIB_PREFETCH (&n_left, sizeof (n_left), STORE);
+ }
+
//Move from available to used ring
- rxvq->used->ring[used_index & qsz_mask].id = desc_chain_head;
- rxvq->used->ring[used_index & qsz_mask].len = desc_len;
- vhost_user_log_dirty_ring (vui, rxvq, ring[used_index & qsz_mask]);
+ rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id = desc_head;
+ rxvq->used->ring[rxvq->last_used_idx & qsz_mask].len = desc_len;
+ vhost_user_log_dirty_ring (vui, rxvq,
+ ring[rxvq->last_used_idx & qsz_mask]);
rxvq->last_avail_idx++;
- used_index++;
+ rxvq->last_used_idx++;
+
+ n_left--; //At the end for error counting when 'goto done' is invoked
}
done:
CLIB_MEMORY_BARRIER ();
- rxvq->used->idx = used_index;
+ rxvq->used->idx = rxvq->last_used_idx;
vhost_user_log_dirty_ring (vui, rxvq, idx);
/* interrupt (call) handling */
- if ((rxvq->callfd > 0) && !(rxvq->avail->flags & 1))
+ if ((rxvq->callfd > -1) && !(rxvq->avail->flags & 1))
{
rxvq->n_since_last_int += n_packets - n_left;
for (q = 0; q < 2; q++)
{
vui->vrings[q].enabled = 0;
+ vui->vrings[q].callfd = -1;
+ vui->vrings[q].kickfd = -1;
}
vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0);
vhost_user_vui_register (vlib_main_t * vm, vhost_user_intf_t * vui)
{
vhost_user_main_t *vum = &vhost_user_main;
-#if DPDK > 0
- dpdk_main_t *dm = &dpdk_main;
int cpu_index;
vlib_thread_main_t *tm = vlib_get_thread_main ();
-#endif
hash_set (vum->vhost_user_interface_index_by_listener_fd, vui->unix_fd,
vui - vum->vhost_user_interfaces);
vui - vum->vhost_user_interfaces);
/* start polling */
-#if DPDK > 0
- cpu_index = dm->input_cpu_first_index +
- (vui - vum->vhost_user_interfaces) % dm->input_cpu_count;
+ cpu_index = vum->input_cpu_first_index +
+ (vui - vum->vhost_user_interfaces) % vum->input_cpu_count;
if (tm->n_vlib_mains == 1)
-#endif
vlib_node_set_state (vm, vhost_user_input_node.index,
VLIB_NODE_STATE_POLLING);
-#if DPDK > 0
else
vlib_node_set_state (vlib_mains[cpu_index], vhost_user_input_node.index,
VLIB_NODE_STATE_POLLING);
-#endif
/* tell process to start polling for sockets */
vlib_process_signal_event (vm, vhost_user_process_node.index, 0, 0);
vui->vrings[q].desc[j].next,
pointer_to_uword (map_guest_mem
(vui,
- vui->vrings[q].
- desc[j].addr)));
+ vui->vrings[q].desc[j].
+ addr)));
}
}
}