return 0;
}
-
-static inline void *
-map_guest_mem (vhost_user_intf_t * vui, uword addr)
+static_always_inline void *
+map_guest_mem (vhost_user_intf_t * vui, uword addr, u32 * hint)
{
- int i;
+ int i = *hint;
+ if (PREDICT_TRUE ((vui->regions[i].guest_phys_addr <= addr) &&
+ ((vui->regions[i].guest_phys_addr +
+ vui->regions[i].memory_size) > addr)))
+ {
+ return (void *) (vui->region_mmap_addr[i] + addr -
+ vui->regions[i].guest_phys_addr);
+ }
+#if __SSE4_2__
+ __m128i rl, rh, al, ah, r;
+ al = _mm_set1_epi64x (addr + 1);
+ ah = _mm_set1_epi64x (addr);
+
+ rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[0]);
+ rl = _mm_cmpgt_epi64 (al, rl);
+ rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[0]);
+ rh = _mm_cmpgt_epi64 (rh, ah);
+ r = _mm_and_si128 (rl, rh);
+
+ rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[2]);
+ rl = _mm_cmpgt_epi64 (al, rl);
+ rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[2]);
+ rh = _mm_cmpgt_epi64 (rh, ah);
+ r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x22);
+
+ rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[4]);
+ rl = _mm_cmpgt_epi64 (al, rl);
+ rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[4]);
+ rh = _mm_cmpgt_epi64 (rh, ah);
+ r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x44);
+
+ rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[6]);
+ rl = _mm_cmpgt_epi64 (al, rl);
+ rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[6]);
+ rh = _mm_cmpgt_epi64 (rh, ah);
+ r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x88);
+
+ r = _mm_shuffle_epi8 (r, _mm_set_epi64x (0, 0x0e060c040a020800));
+ i = __builtin_ctzll (_mm_movemask_epi8 (r));
+
+ if (i < vui->nregions)
+ {
+ *hint = i;
+ return (void *) (vui->region_mmap_addr[i] + addr -
+ vui->regions[i].guest_phys_addr);
+ }
+
+#else
for (i = 0; i < vui->nregions; i++)
{
if ((vui->regions[i].guest_phys_addr <= addr) &&
((vui->regions[i].guest_phys_addr + vui->regions[i].memory_size) >
addr))
{
+ *hint = i;
return (void *) (vui->region_mmap_addr[i] + addr -
vui->regions[i].guest_phys_addr);
}
}
+#endif
DBG_VQ ("failed to map guest mem addr %llx", addr);
+ *hint = 0;
return 0;
}
vui->is_up = 0;
for (q = 0; q < vui->num_vrings; q++)
{
+ if (vui->vrings[q].callfd > -1)
+ {
+ unix_file_t *uf = pool_elt_at_index (unix_main.file_pool,
+ vui->vrings[q].callfd_idx);
+ unix_file_del (&unix_main, uf);
+ }
+
+ if (vui->vrings[q].kickfd > -1)
+ close (vui->vrings[q].kickfd);
+
+ vui->vrings[q].callfd = -1;
+ vui->vrings[q].kickfd = -1;
vui->vrings[q].desc = NULL;
vui->vrings[q].avail = NULL;
vui->vrings[q].used = NULL;
vui->region_mmap_addr[i] = mmap (0, map_sz, PROT_READ | PROT_WRITE,
MAP_SHARED, fds[i], 0);
+ vui->region_guest_addr_lo[i] = vui->regions[i].guest_phys_addr;
+ vui->region_guest_addr_hi[i] = vui->regions[i].guest_phys_addr +
+ vui->regions[i].memory_size;
DBG_SOCK
("map memory region %d addr 0 len 0x%lx fd %d mapped 0x%lx "
}
vui->vrings[msg.state.index].last_used_idx =
+ vui->vrings[msg.state.index].last_avail_idx =
vui->vrings[msg.state.index].used->idx;
/* tell driver that we don't want interrupts */
goto close_socket;
/* if there is old fd, delete it */
- if (vui->vrings[q].callfd)
+ if (vui->vrings[q].callfd > -1)
{
unix_file_t *uf = pool_elt_at_index (unix_main.file_pool,
vui->vrings[q].callfd_idx);
if (number_of_fds != 1)
goto close_socket;
+ if (vui->vrings[q].kickfd > -1)
+ close (vui->vrings[q].kickfd);
+
vui->vrings[q].kickfd = fds[0];
}
else
u16 qsz_mask;
u32 cpu_index, rx_len, drops, flush;
f64 now = vlib_time_now (vm);
+ u32 map_guest_hint_desc = 0;
+ u32 map_guest_hint_indirect = 0;
+ u32 *map_guest_hint_p = &map_guest_hint_desc;
vec_reset_length (vui->d_trace_buffers);
u32 next_desc =
txvq->avail->ring[(txvq->last_avail_idx + 1) & qsz_mask];
void *buffer_addr =
- map_guest_mem (vui, txvq->desc[next_desc].addr);
+ map_guest_mem (vui, txvq->desc[next_desc].addr,
+ &map_guest_hint_desc);
if (PREDICT_TRUE (buffer_addr != 0))
CLIB_PREFETCH (buffer_addr, 64, STORE);
vring_desc_t *desc_table = txvq->desc;
u32 desc_index = desc_current;
+ map_guest_hint_p = &map_guest_hint_desc;
if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
{
- desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr);
+ desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr,
+ &map_guest_hint_desc);
desc_index = 0;
+ map_guest_hint_p = &map_guest_hint_indirect;
if (PREDICT_FALSE (desc_table == 0))
{
error = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
while (1)
{
void *buffer_addr =
- map_guest_mem (vui, desc_table[desc_index].addr);
+ map_guest_mem (vui, desc_table[desc_index].addr,
+ map_guest_hint_p);
if (PREDICT_FALSE (buffer_addr == 0))
{
error = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
}
/* interrupt (call) handling */
- if ((txvq->callfd > 0) && !(txvq->avail->flags & 1))
+ if ((txvq->callfd > -1) && !(txvq->avail->flags & 1))
{
txvq->n_since_last_int += n_rx_packets;
u8 error = VHOST_USER_TX_FUNC_ERROR_NONE;
n_left = n_packets = frame->n_vectors;
+ u32 map_guest_hint_desc = 0;
+ u32 map_guest_hint_indirect = 0;
+ u32 *map_guest_hint_p = &map_guest_hint_desc;
if (PREDICT_FALSE (!vui->is_up))
goto done2;
}
desc_table = rxvq->desc;
+ map_guest_hint_p = &map_guest_hint_desc;
desc_head = desc_index =
rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
if (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT)
}
if (PREDICT_FALSE
(!(desc_table =
- map_guest_mem (vui, rxvq->desc[desc_index].addr))))
+ map_guest_mem (vui, rxvq->desc[desc_index].addr,
+ &map_guest_hint_desc))))
{
error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
goto done;
}
desc_index = 0;
+ map_guest_hint_p = &map_guest_hint_indirect;
}
desc_len = vui->virtio_net_hdr_sz;
if (PREDICT_FALSE
- (!(buffer_addr = map_guest_mem (vui, desc_table[desc_index].addr))))
+ (!(buffer_addr =
+ map_guest_mem (vui, desc_table[desc_index].addr,
+ map_guest_hint_p))))
{
error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
goto done;
}
buffer_len = desc_table[desc_index].len;
- CLIB_PREFETCH (buffer_addr,
- clib_min (buffer_len, 2 * CLIB_CACHE_LINE_BYTES), STORE);
+ CLIB_PREFETCH (buffer_addr, CLIB_CACHE_LINE_BYTES, STORE);
virtio_net_hdr_mrg_rxbuf_t *hdr =
(virtio_net_hdr_mrg_rxbuf_t *) buffer_addr;
desc_index = desc_table[desc_index].next;
if (PREDICT_FALSE
(!(buffer_addr =
- map_guest_mem (vui, desc_table[desc_index].addr))))
+ map_guest_mem (vui, desc_table[desc_index].addr,
+ map_guest_hint_p))))
{
rxvq->last_used_idx -= hdr->num_buffers - 1;
rxvq->last_avail_idx -= hdr->num_buffers - 1;
}
desc_table = rxvq->desc;
+ map_guest_hint_p = &map_guest_hint_desc;
desc_head = desc_index =
rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
if (PREDICT_FALSE
if (PREDICT_FALSE
(!(desc_table =
map_guest_mem (vui,
- rxvq->desc[desc_index].addr))))
+ rxvq->desc[desc_index].addr,
+ &map_guest_hint_desc))))
{
error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
goto done;
}
desc_index = 0;
+ map_guest_hint_p = &map_guest_hint_indirect;
}
if (PREDICT_FALSE
(!(buffer_addr =
- map_guest_mem (vui, desc_table[desc_index].addr))))
+ map_guest_mem (vui, desc_table[desc_index].addr,
+ map_guest_hint_p))))
{
error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
goto done;
}
buffer_len = desc_table[desc_index].len;
- CLIB_PREFETCH (buffer_addr,
- clib_min (buffer_len,
- 2 * CLIB_CACHE_LINE_BYTES), STORE);
+ CLIB_PREFETCH (buffer_addr, CLIB_CACHE_LINE_BYTES, STORE);
}
else
{
vhost_user_log_dirty_ring (vui, rxvq, idx);
/* interrupt (call) handling */
- if ((rxvq->callfd > 0) && !(rxvq->avail->flags & 1))
+ if ((rxvq->callfd > -1) && !(rxvq->avail->flags & 1))
{
rxvq->n_since_last_int += n_packets - n_left;
for (q = 0; q < 2; q++)
{
vui->vrings[q].enabled = 0;
+ vui->vrings[q].callfd = -1;
+ vui->vrings[q].kickfd = -1;
}
vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0);
" ===== ================== ===== ====== ===== ==================\n");
for (j = 0; j < vui->vrings[q].qsz; j++)
{
+ u32 mem_hint = 0;
vlib_cli_output (vm,
" %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n",
j, vui->vrings[q].desc[j].addr,
pointer_to_uword (map_guest_mem
(vui,
vui->vrings[q].desc[j].
- addr)));
+ addr, &mem_hint)));
}
}
}
* CLI functions
*/
-#if DPDK == 0
/* *INDENT-OFF* */
VLIB_CLI_COMMAND (vhost_user_connect_command, static) = {
.path = "create vhost-user",
.function = show_vhost_user_command_fn,
};
/* *INDENT-ON* */
-#endif
static clib_error_t *
vhost_user_config (vlib_main_t * vm, unformat_input_t * input)