vlib_node_registration_t vhost_user_input_node;
#define foreach_vhost_user_tx_func_error \
+ _(NONE, "no error") \
+ _(NOT_READY, "vhost user state error") \
_(PKT_DROP_NOBUF, "tx packet drops (no available descriptors)") \
_(MMAP_FAIL, "mmap failure")
#define foreach_vhost_user_input_func_error \
_(NO_ERROR, "no error") \
+ _(NO_BUFFER, "no available buffer") \
+ _(MMAP_FAIL, "mmap failure") \
_(UNDERSIZED_FRAME, "undersized ethernet frame received (< 14 bytes)")
typedef enum {
}
-static inline void * map_guest_mem(vhost_user_intf_t * vui, u64 addr)
+static inline void * map_guest_mem(vhost_user_intf_t * vui, uword addr)
{
int i;
for (i=0; i<vui->nregions; i++) {
return 0;
}
-static inline void * map_user_mem(vhost_user_intf_t * vui, u64 addr)
+static inline void * map_user_mem(vhost_user_intf_t * vui, uword addr)
{
int i;
for (i=0; i<vui->nregions; i++) {
vui->vrings[q].desc = NULL;
vui->vrings[q].avail = NULL;
vui->vrings[q].used = NULL;
+ vui->vrings[q].log_guest_addr = 0;
+ vui->vrings[q].log_used = 0;
}
unmap_all_mem_regions(vui);
DBG_SOCK("interface ifindex %d disconnected", vui->sw_if_index);
}
+#define VHOST_LOG_PAGE 0x1000
+always_inline void vhost_user_log_dirty_pages(vhost_user_intf_t * vui,
+ u64 addr, u64 len)
+{
+ if (PREDICT_TRUE(vui->log_base_addr == 0
+ || !(vui->features & (1 << FEAT_VHOST_F_LOG_ALL)))) {
+ return;
+ }
+ if (PREDICT_FALSE((addr + len - 1) / VHOST_LOG_PAGE / 8 >= vui->log_size)) {
+ DBG_SOCK("vhost_user_log_dirty_pages(): out of range\n");
+ return;
+ }
+
+ CLIB_MEMORY_BARRIER();
+ u64 page = addr / VHOST_LOG_PAGE;
+ while (page * VHOST_LOG_PAGE < addr + len) {
+ ((u8*)vui->log_base_addr)[page / 8] |= 1 << page % 8;
+ page++;
+ }
+}
+
+#define vhost_user_log_dirty_ring(vui, vq, member) \
+ if (PREDICT_FALSE(vq->log_used)) { \
+ vhost_user_log_dirty_pages(vui, vq->log_guest_addr + offsetof(vring_used_t, member), \
+ sizeof(vq->used->member)); \
+ }
+
static clib_error_t * vhost_user_socket_read (unix_file_t * uf)
{
int n, i;
(cmsg->cmsg_type == SCM_RIGHTS) &&
(cmsg->cmsg_len - CMSG_LEN(0) <= VHOST_MEMORY_MAX_NREGIONS * sizeof(int))) {
number_of_fds = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int);
- memcpy(fds, CMSG_DATA(cmsg), number_of_fds * sizeof(int));
+ clib_memcpy(fds, CMSG_DATA(cmsg), number_of_fds * sizeof(int));
}
/* version 1, no reply bit set*/
msg.flags |= 4;
msg.u64 = (1 << FEAT_VIRTIO_NET_F_MRG_RXBUF) |
- (1 << FEAT_VIRTIO_F_ANY_LAYOUT);
+ (1 << FEAT_VIRTIO_F_ANY_LAYOUT) |
+ (1 << FEAT_VHOST_F_LOG_ALL) |
+ (1 << FEAT_VIRTIO_NET_F_GUEST_ANNOUNCE) |
+ (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES);
msg.u64 &= vui->feature_mask;
msg.size = sizeof(msg.u64);
vui->hw_if_index, msg.u64);
vui->features = msg.u64;
+
if (vui->features & (1 << FEAT_VIRTIO_NET_F_MRG_RXBUF))
vui->virtio_net_hdr_sz = 12;
else
vui->vrings[q].desc = 0;
vui->vrings[q].avail = 0;
vui->vrings[q].used = 0;
+ vui->vrings[q].log_guest_addr = 0;
+ vui->vrings[q].log_used = 0;
}
DBG_SOCK("interface %d disconnected", vui->sw_if_index);
}
unmap_all_mem_regions(vui);
for(i=0; i < msg.memory.nregions; i++) {
- memcpy(&(vui->regions[i]), &msg.memory.regions[i],
+ clib_memcpy(&(vui->regions[i]), &msg.memory.regions[i],
sizeof(vhost_user_memory_region_t));
long page_sz = get_huge_page_size(fds[i]);
goto close_socket;
}
+ vui->vrings[msg.state.index].log_guest_addr = msg.addr.log_guest_addr;
+ vui->vrings[msg.state.index].log_used =
+ (msg.addr.flags & (1 << VHOST_VRING_F_LOG)) ? 1 : 0;
+
+ /* Spec says: If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated,
+ the ring is initialized in an enabled state. */
+
+ if (!(vui->features & (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES))) {
+ vui->vrings[msg.state.index].enabled = 1;
+ }
+
vui->vrings[msg.state.index].last_used_idx =
vui->vrings[msg.state.index].used->idx;
DBG_SOCK("if %d msg VHOST_USER_GET_VRING_BASE idx %d num %d",
vui->hw_if_index, msg.state.index, msg.state.num);
- msg.state.num = vui->vrings[msg.state.index].last_used_idx;
+ /* Spec says: Client must [...] stop ring upon receiving VHOST_USER_GET_VRING_BASE. */
+ vui->vrings[msg.state.index].enabled = 0;
+
+ msg.state.num = vui->vrings[msg.state.index].last_avail_idx;
msg.flags |= 4;
msg.size = sizeof(msg.state);
break;
break;
case VHOST_USER_SET_LOG_BASE:
+ {
DBG_SOCK("if %d msg VHOST_USER_SET_LOG_BASE",
vui->hw_if_index);
+ if (msg.size != sizeof(msg.log)) {
+ DBG_SOCK("invalid msg size for VHOST_USER_SET_LOG_BASE: %d instead of %d",
+ msg.size, sizeof(msg.log));
+ goto close_socket;
+ }
+
+ if (!(vui->protocol_features & (1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD))) {
+ DBG_SOCK("VHOST_USER_PROTOCOL_F_LOG_SHMFD not set but VHOST_USER_SET_LOG_BASE received");
+ goto close_socket;
+ }
+
+ fd = fds[0];
+ /* align size to 2M page */
+ long page_sz = get_huge_page_size(fd);
+ ssize_t map_sz = (msg.log.size + msg.log.offset + page_sz) & ~(page_sz - 1);
+
+ vui->log_base_addr = mmap(0, map_sz, PROT_READ | PROT_WRITE,
+ MAP_SHARED, fd, 0);
+
+ DBG_SOCK("map log region addr 0 len 0x%lx off 0x%lx fd %d mapped 0x%lx",
+ map_sz, msg.log.offset, fd, vui->log_base_addr);
+
+ if (vui->log_base_addr == MAP_FAILED) {
+ clib_warning("failed to map memory. errno is %d", errno);
+ goto close_socket;
+ }
+
+ vui->log_base_addr += msg.log.offset;
+ vui->log_size = msg.log.size;
+
+ msg.flags |= 4;
+ msg.size = sizeof(msg.u64);
+
break;
+ }
case VHOST_USER_SET_LOG_FD:
DBG_SOCK("if %d msg VHOST_USER_SET_LOG_FD",
break;
+ case VHOST_USER_GET_PROTOCOL_FEATURES:
+ DBG_SOCK("if %d msg VHOST_USER_GET_PROTOCOL_FEATURES", vui->hw_if_index);
+
+ msg.flags |= 4;
+ msg.u64 = (1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD);
+ msg.size = sizeof(msg.u64);
+ break;
+
+ case VHOST_USER_SET_PROTOCOL_FEATURES:
+ DBG_SOCK("if %d msg VHOST_USER_SET_PROTOCOL_FEATURES features 0x%lx",
+ vui->hw_if_index, msg.u64);
+
+ vui->protocol_features = msg.u64;
+
+ break;
+
+ case VHOST_USER_SET_VRING_ENABLE:
+ DBG_SOCK("if %d VHOST_USER_SET_VRING_ENABLE, enable: %d",
+ vui->hw_if_index, msg.state.num);
+ vui->vrings[msg.state.index].enabled = msg.state.num;
+ break;
+
default:
DBG_SOCK("unknown vhost-user message %d received. closing socket",
msg.request);
t0->virtqueue = virtqueue;
t0->device_index = vui - vum->vhost_user_interfaces;
#if VHOST_USER_COPY_TX_HDR == 1
- rte_memcpy(&t0->hdr, b0->pre_data, sizeof(virtio_net_hdr_t));
+ clib_memcpy(&t0->hdr, b0->pre_data, sizeof(virtio_net_hdr_t));
#endif
b+=1;
vq->int_deadline = vlib_time_now(vm) + vum->coalesce_time;
}
+
static u32 vhost_user_if_input ( vlib_main_t * vm,
vhost_user_main_t * vum,
vhost_user_intf_t * vui,
{
vhost_user_vring_t * txvq = &vui->vrings[VHOST_NET_VRING_IDX_TX];
vhost_user_vring_t * rxvq = &vui->vrings[VHOST_NET_VRING_IDX_RX];
- uword n_rx_packets = 0;
+ uword n_rx_packets = 0, n_rx_bytes = 0;
uword n_left;
- u32 bi;
u32 n_left_to_next, * to_next;
- u32 next_index = VHOST_USER_RX_NEXT_ETHERNET_INPUT;
- uword n_rx_bytes = 0;
+ u32 next_index = 0;
+ u32 next0;
uword n_trace = vlib_get_trace_count (vm, node);
u16 qsz_mask;
+ u32 cpu_index, rx_len, drops, flush;
f64 now = vlib_time_now (vm);
- u32 cpu_index;
vec_reset_length (vui->d_trace_buffers);
- u32 free_list_index = VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX;
/* no descriptor ptr - bail out */
- if (PREDICT_FALSE(!txvq->desc))
- return 0;
-
- if (PREDICT_FALSE(!txvq->avail))
+ if (PREDICT_FALSE(!txvq->desc || !txvq->avail || !txvq->enabled))
return 0;
/* do we have pending intterupts ? */
if (txvq->avail->idx == txvq->last_avail_idx)
return 0;
- cpu_index = os_get_cpu_number();
-
if (PREDICT_TRUE(txvq->avail->idx > txvq->last_avail_idx))
n_left = txvq->avail->idx - txvq->last_avail_idx;
else /* wrapped */
n_left = (u16) -1 - txvq->last_avail_idx + txvq->avail->idx;
if (PREDICT_FALSE(!vui->admin_up)) {
- /* if intf is admin down, just drop all packets waiting in the ring */
- txvq->last_avail_idx = txvq->last_used_idx = txvq->avail->idx;
- CLIB_MEMORY_BARRIER();
- txvq->used->idx = txvq->last_used_idx;
- vhost_user_send_call(vm, txvq);
-
- return 0;
+ /* if intf is admin down, just drop all packets waiting in the ring */
+ txvq->last_avail_idx = txvq->last_used_idx = txvq->avail->idx;
+ CLIB_MEMORY_BARRIER();
+ txvq->used->idx = txvq->last_used_idx;
+ vhost_user_log_dirty_ring(vui, txvq, idx);
+ vhost_user_send_call(vm, txvq);
+ return 0;
}
- if (PREDICT_FALSE(n_left > txvq->qsz)) {
+ if (PREDICT_FALSE(n_left > txvq->qsz))
return 0;
- }
- if (PREDICT_FALSE(n_left > VLIB_FRAME_SIZE))
+ qsz_mask = txvq->qsz - 1;
+ cpu_index = os_get_cpu_number();
+ drops = 0;
+ flush = 0;
+
+ if (n_left > VLIB_FRAME_SIZE)
n_left = VLIB_FRAME_SIZE;
- /* Make sure we have some RX buffers. */
- {
- uword l = vec_len (vum->rx_buffers[cpu_index]);
- uword n_alloc;
+ /* Allocate some buffers.
+ * Note that buffers that are chained for jumbo
+ * frames are allocated separately using a slower path.
+ * The idea is to be certain to have enough buffers at least
+ * to cycle through the descriptors without having to check for errors.
+ * For jumbo frames, the bottleneck is memory copy anyway.
+ */
+ if (PREDICT_FALSE(!vum->rx_buffers[cpu_index])) {
+ vec_alloc (vum->rx_buffers[cpu_index], VLIB_FRAME_SIZE);
+
+ if (PREDICT_FALSE(!vum->rx_buffers[cpu_index]))
+ flush = n_left; //Drop all input
+ }
- if (l < n_left)
- {
- if (! vum->rx_buffers[cpu_index]) {
- vec_alloc (vum->rx_buffers[cpu_index], 2 * VLIB_FRAME_SIZE );
- }
+ if (PREDICT_FALSE(_vec_len(vum->rx_buffers[cpu_index]) < n_left)) {
+ _vec_len(vum->rx_buffers[cpu_index]) +=
+ vlib_buffer_alloc_from_free_list(vm, vum->rx_buffers[cpu_index] + _vec_len(vum->rx_buffers[cpu_index]),
+ VLIB_FRAME_SIZE - _vec_len(vum->rx_buffers[cpu_index]),
+ VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
- n_alloc = vlib_buffer_alloc_from_free_list
- (vm, vum->rx_buffers[cpu_index] + l, 2 * VLIB_FRAME_SIZE - l,
- free_list_index);
- if (n_alloc == 0)
- return 0;
- _vec_len (vum->rx_buffers[cpu_index]) = l + n_alloc;
- }
+ if (PREDICT_FALSE(n_left > _vec_len(vum->rx_buffers[cpu_index])))
+ flush = n_left - _vec_len(vum->rx_buffers[cpu_index]);
}
- qsz_mask = txvq->qsz - 1;
+ if (PREDICT_FALSE(flush)) {
+ //Remove some input buffers
+ drops += flush;
+ n_left -= flush;
+ vlib_error_count(vm, vhost_user_input_node.index,
+ VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, flush);
+ while (flush) {
+ u16 desc_chain_head = txvq->avail->ring[txvq->last_avail_idx & qsz_mask];
+ txvq->last_avail_idx++;
+ txvq->used->ring[txvq->last_used_idx & qsz_mask].id = desc_chain_head;
+ txvq->used->ring[txvq->last_used_idx & qsz_mask].len = 0;
+ vhost_user_log_dirty_ring(vui, txvq, ring[txvq->last_used_idx & qsz_mask]);
+ txvq->last_used_idx++;
+ flush--;
+ }
+ }
+ rx_len = vec_len(vum->rx_buffers[cpu_index]); //vector might be null
while (n_left > 0) {
vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
while (n_left > 0 && n_left_to_next > 0) {
- vlib_buffer_t * b;
- u16 desc_chain_head = txvq->avail->ring[txvq->last_avail_idx & qsz_mask];
- u16 desc_current = desc_chain_head;
- uword i_rx = vec_len (vum->rx_buffers[cpu_index]) - 1;
-
- bi = vum->rx_buffers[cpu_index][i_rx];
- b = vlib_get_buffer (vm, bi);
+ vlib_buffer_t *b_head, *b_current;
+ u32 bi_head, bi_current;
+ u16 desc_chain_head, desc_current;
+ u8 error = VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR;
- vlib_prefetch_buffer_with_index (vm, vum->rx_buffers[cpu_index][i_rx-1], STORE);
+ desc_chain_head = desc_current = txvq->avail->ring[txvq->last_avail_idx & qsz_mask];
+ bi_head = bi_current = vum->rx_buffers[cpu_index][--rx_len];
+ b_head = b_current = vlib_get_buffer (vm, bi_head);
+ vlib_buffer_chain_init(b_head);
uword offset;
- if (PREDICT_TRUE(vui->is_any_layout))
+ if (PREDICT_TRUE(vui->is_any_layout) ||
+ !(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT)) {
+ /* ANYLAYOUT or single buffer */
offset = vui->virtio_net_hdr_sz;
- else if (!(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT))
- /* WSA case, no ANYLAYOUT but single buffer */
- offset = vui->virtio_net_hdr_sz;
- else
+ } else {
/* CSR case without ANYLAYOUT, skip 1st buffer */
offset = txvq->desc[desc_current].len;
-
- uword ptr=0;
+ }
while(1) {
void * buffer_addr = map_guest_mem(vui, txvq->desc[desc_current].addr);
- CLIB_PREFETCH (&txvq->desc[txvq->desc[desc_current].next], sizeof (vring_desc_t), READ);
+ if (PREDICT_FALSE(buffer_addr == 0)) {
+ error = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
+ break;
+ }
#if VHOST_USER_COPY_TX_HDR == 1
- if (PREDICT_TRUE(offset)) {
- rte_memcpy(b->pre_data, buffer_addr, sizeof(virtio_net_hdr_t)); /* 12 byte hdr is not used on tx */
- }
+ if (PREDICT_TRUE(offset))
+ clib_memcpy(b->pre_data, buffer_addr, sizeof(virtio_net_hdr_t)); /* 12 byte hdr is not used on tx */
#endif
if (txvq->desc[desc_current].len > offset) {
u16 len = txvq->desc[desc_current].len - offset;
+ u16 copied = vlib_buffer_chain_append_data_with_alloc(vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX,
+ b_head, &b_current, buffer_addr + offset, len);
- if (PREDICT_FALSE(len > VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES))
- len = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES;
-
- rte_memcpy(vlib_buffer_get_current (b) + ptr,
- buffer_addr + offset, len);
+ if (copied != len) {
+ error = VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER;
+ break;
+ }
}
- ptr += txvq->desc[desc_current].len - offset;
offset = 0;
/* if next flag is set, take next desc in the chain */
break;
}
+ /* consume the descriptor and return it as used */
txvq->last_avail_idx++;
-
- /* returning buffer */
txvq->used->ring[txvq->last_used_idx & qsz_mask].id = desc_chain_head;
- txvq->used->ring[txvq->last_used_idx & qsz_mask].len = ptr + vui->virtio_net_hdr_sz;
-
+ txvq->used->ring[txvq->last_used_idx & qsz_mask].len = 0;
+ vhost_user_log_dirty_ring(vui, txvq, ring[txvq->last_used_idx & qsz_mask]);
txvq->last_used_idx++;
- b->current_length = ptr;
-
- if(PREDICT_FALSE(b->current_length < 14)) {
- vlib_error_count(vm, vhost_user_input_node.index,
- VHOST_USER_INPUT_FUNC_ERROR_UNDERSIZED_FRAME, 1);
- goto skip_frame;
+ if(PREDICT_FALSE(b_head->current_length < 14 &&
+ error == VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR)) {
+ error = VHOST_USER_INPUT_FUNC_ERROR_UNDERSIZED_FRAME;
}
- b->flags = 0;
- b->current_data = 0;
- b->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
- n_rx_bytes += ptr;
- _vec_len (vum->rx_buffers[cpu_index]) = i_rx;
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b_head);
- /*
- * Turn this on if you run into
- * "bad monkey" contexts, and you want to know exactly
- * which nodes they've visited... See .../vlib/vlib/buffer.h
- */
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b);
+ vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index;
+ vnet_buffer (b_head)->sw_if_index[VLIB_TX] = (u32)~0;
+ b_head->error = node->errors[error];
- vnet_buffer (b)->sw_if_index[VLIB_RX] = vui->sw_if_index;
- vnet_buffer (b)->sw_if_index[VLIB_TX] = (u32)~0;
- b->error = node->errors[0];
+ if (PREDICT_FALSE (n_trace > n_rx_packets))
+ vec_add1 (vui->d_trace_buffers, bi_head);
+
+ if (PREDICT_FALSE(error)) {
+ drops++;
+ next0 = VHOST_USER_RX_NEXT_DROP;
+ } else {
+ n_rx_bytes += b_head->current_length + b_head->total_length_not_including_first_buffer;
+ n_rx_packets++;
+ next0 = VHOST_USER_RX_NEXT_ETHERNET_INPUT;
+ }
- to_next[0] = bi;
+ to_next[0] = bi_head;
to_next++;
n_left_to_next--;
vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
- to_next, n_left_to_next,
- bi, next_index);
-
- if (PREDICT_FALSE (n_trace > n_rx_packets))
- vec_add1 (vui->d_trace_buffers, bi);
-
- n_rx_packets++;
-skip_frame:
- n_left--;
+ to_next, n_left_to_next,
+ bi_head, next0);
+ n_left--;
}
- /* give buffers back to driver */
- CLIB_MEMORY_BARRIER();
- txvq->used->idx = txvq->last_used_idx;
-
vlib_put_next_frame (vm, node, next_index, n_left_to_next);
}
+ if (PREDICT_TRUE(vum->rx_buffers[cpu_index] != 0))
+ _vec_len(vum->rx_buffers[cpu_index]) = rx_len;
+
+ /* give buffers back to driver */
+ CLIB_MEMORY_BARRIER();
+ txvq->used->idx = txvq->last_used_idx;
+ vhost_user_log_dirty_ring(vui, txvq, idx);
+
if (PREDICT_FALSE (vec_len (vui->d_trace_buffers) > 0))
{
- vhost_user_rx_trace (vm, node, vui, VHOST_NET_VRING_IDX_TX);
- vlib_set_trace_count (vm, node, n_trace - vec_len (vui->d_trace_buffers));
+ vhost_user_rx_trace (vm, node, vui, VHOST_NET_VRING_IDX_TX);
+ vlib_set_trace_count (vm, node, n_trace - vec_len (vui->d_trace_buffers));
}
- /* if no packets received we're done */
- if(!n_rx_packets)
- return 0;
-
/* interrupt (call) handling */
if((txvq->callfd > 0) && !(txvq->avail->flags & 1)) {
txvq->n_since_last_int += n_rx_packets;
vhost_user_send_call(vm, txvq);
}
+ if (PREDICT_FALSE(drops)) {
+ vlib_increment_simple_counter
+ (vnet_main.interface_main.sw_if_counters
+ + VNET_INTERFACE_COUNTER_DROP, os_get_cpu_number(),
+ vui->sw_if_index, drops);
+ }
+
/* increase rx counters */
vlib_increment_combined_counter
(vnet_main.interface_main.combined_sw_if_counters
},
};
+VLIB_NODE_FUNCTION_MULTIARCH (vhost_user_input_node, vhost_user_input)
+
static uword
vhost_user_intfc_tx (vlib_main_t * vm,
vlib_node_runtime_t * node,
u16 used_index;
vhost_user_main_t * vum = &vhost_user_main;
uword n_packets = 0;
- uword n_avail_desc;
vnet_interface_output_runtime_t * rd = (void *) node->runtime_data;
vhost_user_intf_t * vui = vec_elt_at_index (vum->vhost_user_interfaces, rd->dev_instance);
vhost_user_vring_t * rxvq = &vui->vrings[VHOST_NET_VRING_IDX_RX];
u16 qsz_mask;
+ u8 error = VHOST_USER_TX_FUNC_ERROR_NONE;
if (PREDICT_FALSE(!vui->is_up))
goto done2;
- if (PREDICT_FALSE(!rxvq->desc))
+ if (PREDICT_FALSE(!rxvq->desc || !rxvq->avail || vui->sock_errno != 0 || !rxvq->enabled)) {
+ error = VHOST_USER_TX_FUNC_ERROR_NOT_READY;
goto done2;
-
- if (PREDICT_FALSE(!rxvq->avail))
- goto done2;
+ }
if (PREDICT_FALSE(vui->lockp != 0))
{
;
}
-
/* only bit 0 of avail.flags is used so we don't want to deal with this
interface if any other bit is set */
- if (PREDICT_FALSE(rxvq->avail->flags & 0xFFFE))
- goto done2;
-
- if (PREDICT_FALSE((rxvq->avail->idx == rxvq->last_avail_idx) ||
- vui->sock_errno != 0)) {
- vlib_simple_counter_main_t * cm;
- vnet_main_t * vnm = vnet_get_main();
-
- cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
- VNET_INTERFACE_COUNTER_TX_ERROR);
- vlib_increment_simple_counter (cm, os_get_cpu_number(),
- 0, frame->n_vectors);
-
- vlib_error_count (vm, node->node_index,
- VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF,
- frame->n_vectors);
+ if (PREDICT_FALSE(rxvq->avail->flags & 0xFFFE)) {
+ error = VHOST_USER_TX_FUNC_ERROR_NOT_READY;
+ goto done2;
+ }
+
+ if (PREDICT_FALSE((rxvq->avail->idx == rxvq->last_avail_idx))) {
+ error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
goto done2;
}
- if (PREDICT_TRUE(rxvq->avail->idx > rxvq->last_avail_idx))
- n_avail_desc = rxvq->avail->idx - rxvq->last_avail_idx;
- else /* wrapped */
- n_avail_desc = (u16) -1 - rxvq->last_avail_idx + rxvq->avail->idx;
-
- DBG_VQ("rxvq->avail->idx %d rxvq->last_avail_idx %d n_avail_desc %d",
- rxvq->avail->idx, rxvq->last_avail_idx, n_avail_desc);
-
n_left = n_packets = frame->n_vectors;
- if (PREDICT_FALSE(n_packets > n_avail_desc)) {
- vlib_simple_counter_main_t * cm;
- vnet_main_t * vnm = vnet_get_main();
-
- cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
- VNET_INTERFACE_COUNTER_TX_ERROR);
- vlib_increment_simple_counter (cm, os_get_cpu_number(),
- 0, frame->n_vectors);
-
- vlib_error_count (vm, node->node_index,
- VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF,
- n_packets - n_avail_desc);
- n_left = n_packets = n_avail_desc;
- }
-
used_index = rxvq->used->idx;
qsz_mask = rxvq->qsz - 1; /* qsz is always power of 2 */
- while (n_left >= 4)
- {
- vlib_buffer_t * b0, * b1;
- u16 desc_chain_head0,desc_chain_head1;
- u16 desc_current0,desc_current1;
- uword offset0, offset1;
- u16 bytes_left0, bytes_left1;
- void *buffer_addr0, *buffer_addr1;
-
- vlib_prefetch_buffer_with_index (vm, buffers[2], LOAD);
- vlib_prefetch_buffer_with_index (vm, buffers[3], LOAD);
-
- b0 = vlib_get_buffer (vm, buffers[0]);
- b1 = vlib_get_buffer (vm, buffers[1]);
- buffers+=2;
- n_left-=2;
-
- desc_current0 = desc_chain_head0 = rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
- desc_current1 = desc_chain_head1 = rxvq->avail->ring[(rxvq->last_avail_idx+1) & qsz_mask];
-
- offset0 = vui->virtio_net_hdr_sz;
-
- offset1 = vui->virtio_net_hdr_sz;
-
- bytes_left0 = b0->current_length;
- bytes_left1 = b1->current_length;
-
- buffer_addr0 = map_guest_mem(vui, rxvq->desc[desc_current0].addr);
- buffer_addr1 = map_guest_mem(vui, rxvq->desc[desc_current1].addr);
-
- if (PREDICT_FALSE(!buffer_addr0)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- goto done;
- }
- if (PREDICT_FALSE(!buffer_addr1)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- goto done;
- }
-
- virtio_net_hdr_mrg_rxbuf_t * hdr0 = (virtio_net_hdr_mrg_rxbuf_t *) buffer_addr0;
- virtio_net_hdr_mrg_rxbuf_t * hdr1 = (virtio_net_hdr_mrg_rxbuf_t *) buffer_addr1;
- hdr0->hdr.flags = 0;
- hdr1->hdr.flags = 0;
- hdr0->hdr.gso_type = 0;
- hdr1->hdr.gso_type = 0;
-
- if (vui->virtio_net_hdr_sz == 12) {
- hdr0->num_buffers = 1;
- hdr1->num_buffers = 1;
- }
-
- buffer_addr0 += offset0;
- buffer_addr1 += offset1;
-
- if (PREDICT_FALSE(!vui->is_any_layout && rxvq->desc[desc_current0].flags & VIRTQ_DESC_F_NEXT))
- rxvq->desc[desc_current0].len = vui->virtio_net_hdr_sz;
-
- if (PREDICT_FALSE(!vui->is_any_layout && rxvq->desc[desc_current1].flags & VIRTQ_DESC_F_NEXT))
- rxvq->desc[desc_current1].len = vui->virtio_net_hdr_sz;
-
- while(1) {
- if (rxvq->desc[desc_current0].len - offset0 > 0 ) {
- u16 bytes_to_copy = bytes_left0 > (rxvq->desc[desc_current0].len - offset0) ? (rxvq->desc[desc_current0].len - offset0) : bytes_left0;
- rte_memcpy(buffer_addr0, vlib_buffer_get_current (b0) + b0->current_length - bytes_left0, bytes_to_copy);
- bytes_left0 -= bytes_to_copy;
- }
-
- if (rxvq->desc[desc_current0].flags & VIRTQ_DESC_F_NEXT ) {
- offset0 = 0;
- desc_current0 = rxvq->desc[desc_current1].next;
- buffer_addr0 = map_guest_mem(vui, rxvq->desc[desc_current0].addr);
- if (PREDICT_FALSE(!buffer_addr0)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- goto done;
- }
- }
- else
- break;
- }
-
- while(1) {
- if (rxvq->desc[desc_current1].len - offset1 > 0 ) {
- u16 bytes_to_copy = bytes_left1 > (rxvq->desc[desc_current1].len - offset1) ? (rxvq->desc[desc_current1].len - offset1) : bytes_left1;
- rte_memcpy(buffer_addr1, vlib_buffer_get_current (b1) + b1->current_length - bytes_left1, bytes_to_copy);
- bytes_left1 -= bytes_to_copy;
- }
-
- if (rxvq->desc[desc_current1].flags & VIRTQ_DESC_F_NEXT ) {
- offset1 = 0;
- desc_current1 = rxvq->desc[desc_current1].next;
- buffer_addr1 = map_guest_mem(vui, rxvq->desc[desc_current1].addr);
- if (PREDICT_FALSE(!buffer_addr1)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- goto done;
- }
- }
- else
- break;
- }
-
- rxvq->used->ring[used_index & qsz_mask].id = desc_chain_head0;
- rxvq->used->ring[used_index & qsz_mask].len = b0->current_length + vui->virtio_net_hdr_sz;
- used_index+=1;
- rxvq->used->ring[used_index & qsz_mask].id = desc_chain_head1;
- rxvq->used->ring[used_index & qsz_mask].len = b1->current_length + vui->virtio_net_hdr_sz;
- used_index+=1;
- rxvq->last_avail_idx+=2;
- }
-
while (n_left > 0)
{
- vlib_buffer_t * b0;
- u16 desc_chain_head;
- u16 desc_current;
+ vlib_buffer_t *b0, *current_b0;
+ u16 desc_chain_head, desc_current, desc_len;
void *buffer_addr;
+ uword offset;
+
+ if (n_left >= 2)
+ vlib_prefetch_buffer_with_index (vm, buffers[1], LOAD);
b0 = vlib_get_buffer (vm, buffers[0]);
buffers++;
n_left--;
- desc_chain_head = rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
- desc_current = desc_chain_head;
-
- uword offset = vui->virtio_net_hdr_sz;
+ if (PREDICT_FALSE(rxvq->last_avail_idx == rxvq->avail->idx)) {
+ error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
+ goto done;
+ }
- u16 bytes_left = b0->current_length;
- buffer_addr = map_guest_mem(vui, rxvq->desc[desc_current].addr);
- if (PREDICT_FALSE(!buffer_addr)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
+ desc_current = desc_chain_head = rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
+ offset = vui->virtio_net_hdr_sz;
+ desc_len = offset;
+ if (PREDICT_FALSE(!(buffer_addr = map_guest_mem(vui, rxvq->desc[desc_current].addr)))) {
+ error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
goto done;
}
+ CLIB_PREFETCH(buffer_addr, clib_min(rxvq->desc[desc_current].len,
+ 4*CLIB_CACHE_LINE_BYTES), STORE);
virtio_net_hdr_mrg_rxbuf_t * hdr = (virtio_net_hdr_mrg_rxbuf_t *) buffer_addr;
hdr->hdr.flags = 0;
hdr->hdr.gso_type = 0;
- if (vui->virtio_net_hdr_sz == 12) {
+ vhost_user_log_dirty_pages(vui, rxvq->desc[desc_current].addr, vui->virtio_net_hdr_sz);
+
+ if (vui->virtio_net_hdr_sz == 12)
hdr->num_buffers = 1;
- }
+ u16 bytes_left = b0->current_length;
buffer_addr += offset;
+ current_b0 = b0;
- if (PREDICT_FALSE(!vui->is_any_layout && rxvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT))
+ //FIXME: This was in the code but I don't think it is valid
+ /*if (PREDICT_FALSE(!vui->is_any_layout && (rxvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT))) {
rxvq->desc[desc_current].len = vui->virtio_net_hdr_sz;
+ }*/
while(1) {
- if (rxvq->desc[desc_current].len - offset > 0 ) {
- u16 bytes_to_copy = bytes_left > (rxvq->desc[desc_current].len - offset) ? (rxvq->desc[desc_current].len - offset) : bytes_left;
- rte_memcpy(buffer_addr, vlib_buffer_get_current (b0) + b0->current_length - bytes_left, bytes_to_copy);
- bytes_left -= bytes_to_copy;
+ if (!bytes_left) { //Get new input
+ if (current_b0->flags & VLIB_BUFFER_NEXT_PRESENT) {
+ current_b0 = vlib_get_buffer(vm, current_b0->next_buffer);
+ bytes_left = current_b0->current_length;
+ } else {
+ //End of packet
+ break;
+ }
}
- if (rxvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT ) {
- offset = 0;
- desc_current = rxvq->desc[desc_current].next;
- buffer_addr = map_guest_mem(vui, rxvq->desc[desc_current].addr);
- if (PREDICT_FALSE(!buffer_addr)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
+ if (rxvq->desc[desc_current].len <= offset) { //Get new output
+ if (rxvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) {
+ offset = 0;
+ desc_current = rxvq->desc[desc_current].next;
+ if (PREDICT_FALSE(!(buffer_addr = map_guest_mem(vui, rxvq->desc[desc_current].addr)))) {
+ used_index -= hdr->num_buffers - 1;
+ rxvq->last_avail_idx -= hdr->num_buffers - 1;
+ error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
+ goto done;
+ }
+ } else if (vui->virtio_net_hdr_sz == 12) { //MRG is available
+
+ //Move from available to used buffer
+ rxvq->used->ring[used_index & qsz_mask].id = desc_chain_head;
+ rxvq->used->ring[used_index & qsz_mask].len = desc_len;
+ vhost_user_log_dirty_ring(vui, rxvq, ring[used_index & qsz_mask]);
+ rxvq->last_avail_idx++;
+ used_index++;
+ hdr->num_buffers++;
+
+ if (PREDICT_FALSE(rxvq->last_avail_idx == rxvq->avail->idx)) {
+ //Dequeue queued descriptors for this packet
+ used_index -= hdr->num_buffers - 1;
+ rxvq->last_avail_idx -= hdr->num_buffers - 1;
+ error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
+ goto done;
+ }
+
+ //Look at next one
+ desc_chain_head = rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
+ desc_current = desc_chain_head;
+ desc_len = 0;
+ offset = 0;
+ if (PREDICT_FALSE(!(buffer_addr = map_guest_mem(vui, rxvq->desc[desc_current].addr)))) {
+ //Dequeue queued descriptors for this packet
+ used_index -= hdr->num_buffers - 1;
+ rxvq->last_avail_idx -= hdr->num_buffers - 1;
+ error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
+ goto done;
+ }
+ } else {
+ error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
goto done;
}
}
- else
- break;
+
+ u16 bytes_to_copy = bytes_left > (rxvq->desc[desc_current].len - offset) ? (rxvq->desc[desc_current].len - offset) : bytes_left;
+ clib_memcpy(buffer_addr, vlib_buffer_get_current (current_b0) + current_b0->current_length - bytes_left, bytes_to_copy);
+
+ vhost_user_log_dirty_pages(vui, rxvq->desc[desc_current].addr + offset, bytes_to_copy);
+ bytes_left -= bytes_to_copy;
+ offset += bytes_to_copy;
+ buffer_addr += bytes_to_copy;
+ desc_len += bytes_to_copy;
}
+ //Move from available to used ring
rxvq->used->ring[used_index & qsz_mask].id = desc_chain_head;
- rxvq->used->ring[used_index & qsz_mask].len = b0->current_length + vui->virtio_net_hdr_sz;
+ rxvq->used->ring[used_index & qsz_mask].len = desc_len;
+ vhost_user_log_dirty_ring(vui, rxvq, ring[used_index & qsz_mask]);
- used_index++;
rxvq->last_avail_idx++;
+ used_index++;
}
done:
CLIB_MEMORY_BARRIER();
rxvq->used->idx = used_index;
+ vhost_user_log_dirty_ring(vui, rxvq, idx);
/* interrupt (call) handling */
if((rxvq->callfd > 0) && !(rxvq->avail->flags & 1)) {
if (PREDICT_FALSE(vui->lockp != 0))
*vui->lockp = 0;
+ if (PREDICT_FALSE(n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE)) {
+ vlib_error_count(vm, node->node_index, error, n_left);
+ vlib_increment_simple_counter
+ (vnet_main.interface_main.sw_if_counters
+ + VNET_INTERFACE_COUNTER_DROP,
+ os_get_cpu_number(),
+ vui->sw_if_index,
+ n_left);
+ }
+
vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors);
return frame->n_vectors;
}
.format_device_name = format_vhost_user_interface_name,
.name_renumber = vhost_user_name_renumber,
.admin_up_down_function = vhost_user_interface_admin_up_down,
+ .no_flatten_output_chains = 1,
};
+VLIB_DEVICE_TX_FUNCTION_MULTIARCH (vhost_user_dev_class,
+ vhost_user_intfc_tx)
+
static uword
vhost_user_process (vlib_main_t * vm,
vlib_node_runtime_t * rt,
// create ethernet interface for vhost user intf
static void vhost_user_create_ethernet(vnet_main_t * vnm, vlib_main_t * vm,
- vhost_user_intf_t *vui)
+ vhost_user_intf_t *vui, u8 *hwaddress)
{
vhost_user_main_t * vum = &vhost_user_main;
u8 hwaddr[6];
clib_error_t * error;
/* create hw and sw interface */
- {
+ if (hwaddress) {
+ clib_memcpy(hwaddr, hwaddress, 6);
+ } else {
f64 now = vlib_time_now(vm);
u32 rnd;
rnd = (u32) (now * 1e6);
rnd = random_u32 (&rnd);
- memcpy (hwaddr+2, &rnd, sizeof(rnd));
+ clib_memcpy (hwaddr+2, &rnd, sizeof(rnd));
hwaddr[0] = 2;
hwaddr[1] = 0xfe;
}
0 /* flag change */);
if (error)
clib_error_report (error);
+
+ vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, vui->hw_if_index);
+ hi->max_l3_packet_bytes[VLIB_RX] = hi->max_l3_packet_bytes[VLIB_TX] = 9000;
}
// initialize vui with specified attributes
vnet_sw_interface_t * sw;
sw = vnet_get_hw_sw_interface (vnm, vui->hw_if_index);
vlib_thread_main_t * tm = vlib_get_thread_main();
+ int q;
vui->unix_fd = sockfd;
vui->sw_if_index = sw->sw_if_index;
vui->feature_mask = feature_mask;
vui->active = 1;
vui->unix_file_index = ~0;
+ vui->log_base_addr = 0;
+
+ for (q = 0; q < 2; q++) {
+ vui->vrings[q].enabled = 0;
+ }
vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0);
u8 is_server,
u32 * sw_if_index,
u64 feature_mask,
- u8 renumber, u32 custom_dev_instance)
+ u8 renumber, u32 custom_dev_instance,
+ u8 *hwaddr)
{
vhost_user_intf_t * vui = NULL;
dpdk_main_t * dm = &dpdk_main;
vui = vhost_user_vui_new ();
ASSERT(vui != NULL);
- vhost_user_create_ethernet (vnm, vm, vui);
+ vhost_user_create_ethernet (vnm, vm, vui, hwaddr);
vhost_user_vui_init (vnm, vui, sockfd, sock_filename, is_server,
feature_mask, &sw_if_idx);
u64 feature_mask = (u64)~0;
u8 renumber = 0;
u32 custom_dev_instance = ~0;
+ u8 hwaddr[6];
+ u8 *hw = NULL;
/* Get a line of input. */
if (! unformat_user (input, unformat_line_input, line_input))
is_server = 1;
else if (unformat (line_input, "feature-mask 0x%llx", &feature_mask))
;
+ else if (unformat (line_input, "hwaddr %U", unformat_ethernet_address, hwaddr))
+ hw = hwaddr;
else if (unformat (line_input, "renumber %d", &custom_dev_instance)) {
renumber = 1;
}
vhost_user_create_if(vnm, vm, (char *)sock_filename,
is_server, &sw_if_index, feature_mask,
- renumber, custom_dev_instance);
+ renumber, custom_dev_instance, hw);
vec_free(sock_filename);
vui->regions[j].memory_size,
vui->regions[j].userspace_addr,
vui->regions[j].mmap_offset,
- (u64) vui->region_mmap_addr[j]);
+ pointer_to_uword( vui->region_mmap_addr[j]) );
}
for (q = 0; q < vui->num_vrings; q++) {
vlib_cli_output(vm, "\n Virtqueue %d\n", q);
vui->vrings[q].desc[j].len,
vui->vrings[q].desc[j].flags,
vui->vrings[q].desc[j].next,
- (u64) map_guest_mem(vui, vui->vrings[q].desc[j].addr));}
+ pointer_to_uword(map_guest_mem(vui, vui->vrings[q].desc[j].addr)));}
}
}
vlib_cli_output (vm, "\n");
Code Review / vpp.git / blobdiff