*/
#define VHOST_USER_RX_COPY_THRESHOLD 64
-vlib_node_registration_t vhost_user_input_node;
+extern vlib_node_registration_t vhost_user_input_node;
#define foreach_vhost_user_input_func_error \
_(NO_ERROR, "no error") \
out:
txvq->last_avail_idx = last_avail_idx;
txvq->last_used_idx = last_used_idx;
- CLIB_MEMORY_BARRIER ();
+ CLIB_MEMORY_STORE_BARRIER ();
txvq->used->idx = txvq->last_used_idx;
vhost_user_log_dirty_ring (vui, txvq, idx);
return discarded_packets;
/*
* In case of overflow, we need to rewind the array of allocated buffers.
*/
-static __clib_unused void
+static_always_inline void
vhost_user_input_rewind_buffers (vlib_main_t * vm,
vhost_cpu_t * cpu, vlib_buffer_t * b_head)
{
cpu->rx_buffers_len++;
}
-static __clib_unused u32
+static_always_inline u32
vhost_user_if_input (vlib_main_t * vm,
vhost_user_main_t * vum,
vhost_user_intf_t * vui,
u32 n_left_to_next, *to_next;
u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
u32 n_trace = vlib_get_trace_count (vm, node);
+ u32 buffer_data_size = vlib_buffer_get_default_data_size (vm);
u32 map_hint = 0;
vhost_cpu_t *cpu = &vum->cpus[vm->thread_index];
u16 copy_len = 0;
{
u32 curr_len = cpu->rx_buffers_len;
cpu->rx_buffers_len +=
- vlib_buffer_alloc_from_free_list (vm, cpu->rx_buffers + curr_len,
- VHOST_USER_RX_BUFFERS_N - curr_len,
- VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
+ vlib_buffer_alloc (vm, cpu->rx_buffers + curr_len,
+ VHOST_USER_RX_BUFFERS_N - curr_len);
if (PREDICT_FALSE
(cpu->rx_buffers_len < VHOST_USER_RX_BUFFER_STARVATION))
}
/* Get more output if necessary. Or end of packet. */
- if (PREDICT_FALSE
- (b_current->current_length == VLIB_BUFFER_DATA_SIZE))
+ if (PREDICT_FALSE (b_current->current_length == buffer_data_size))
{
if (PREDICT_FALSE (cpu->rx_buffers_len == 0))
{
vhost_copy_t *cpy = &cpu->copy[copy_len];
copy_len++;
u32 desc_data_l = desc_table[desc_current].len - desc_data_offset;
- cpy->len = VLIB_BUFFER_DATA_SIZE - b_current->current_length;
+ cpy->len = buffer_data_size - b_current->current_length;
cpy->len = (cpy->len > desc_data_l) ? desc_data_l : cpy->len;
cpy->dst = (uword) (vlib_buffer_get_current (b_current) +
b_current->current_length);
copy_len = 0;
/* give buffers back to driver */
- CLIB_MEMORY_BARRIER ();
+ CLIB_MEMORY_STORE_BARRIER ();
txvq->used->idx = last_used_idx;
vhost_user_log_dirty_ring (vui, txvq, idx);
}
}
/* give buffers back to driver */
- CLIB_MEMORY_BARRIER ();
+ CLIB_MEMORY_STORE_BARRIER ();
txvq->used->idx = txvq->last_used_idx;
vhost_user_log_dirty_ring (vui, txvq, idx);