From 9282538f0984590369d8096704de61301889d8b0 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Wed, 21 Nov 2018 10:03:44 +0100 Subject: [PATCH] vhost-user: simplify input loop Change-Id: I6e6963882825e83d8da3a460be35c7349e107777 Signed-off-by: Damjan Marion --- src/vnet/devices/virtio/vhost_user_input.c | 355 ++++++++++++++--------------- 1 file changed, 174 insertions(+), 181 deletions(-) diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c index 87a01cefd5b..4fdfbd5d6e0 100644 --- a/src/vnet/devices/virtio/vhost_user_input.c +++ b/src/vnet/devices/virtio/vhost_user_input.c @@ -382,223 +382,216 @@ vhost_user_if_input (vlib_main_t * vm, &next_index, 0); } + vlib_get_new_next_frame (vm, node, next_index, to_next, n_left_to_next); + + if (next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT) + { + /* give some hints to ethernet-input */ + vlib_next_frame_t *nf; + vlib_frame_t *f; + ethernet_input_frame_t *ef; + nf = vlib_node_runtime_get_next_frame (vm, node, next_index); + f = vlib_get_frame (vm, nf->frame_index); + f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX; + + ef = vlib_frame_scalar_args (f); + ef->sw_if_index = vui->sw_if_index; + ef->hw_if_index = vui->hw_if_index; + } + while (n_left > 0) { - vlib_get_new_next_frame (vm, node, next_index, to_next, n_left_to_next); + vlib_buffer_t *b_head, *b_current; + u32 bi_current; + u16 desc_current; + u32 desc_data_offset; + vring_desc_t *desc_table = txvq->desc; - if (next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT) + if (PREDICT_FALSE (cpu->rx_buffers_len <= 1)) { - /* give some hints to ethernet-input */ - vlib_next_frame_t *nf; - vlib_frame_t *f; - ethernet_input_frame_t *ef; - nf = vlib_node_runtime_get_next_frame (vm, node, next_index); - f = vlib_get_frame (vm, nf->frame_index); - f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX; - - ef = vlib_frame_scalar_args (f); - ef->sw_if_index = vui->sw_if_index; - ef->hw_if_index = vui->hw_if_index; + /* Not enough rx_buffers + * Note: We yeld on 1 so we don't need to do an additional + * check for the next buffer prefetch. + */ + n_left = 0; + break; } - while (n_left > 0 && n_left_to_next > 0) + desc_current = txvq->avail->ring[txvq->last_avail_idx & txvq->qsz_mask]; + cpu->rx_buffers_len--; + bi_current = cpu->rx_buffers[cpu->rx_buffers_len]; + b_head = b_current = vlib_get_buffer (vm, bi_current); + to_next[0] = bi_current; //We do that now so we can forget about bi_current + to_next++; + n_left_to_next--; + + vlib_prefetch_buffer_with_index + (vm, cpu->rx_buffers[cpu->rx_buffers_len - 1], LOAD); + + /* Just preset the used descriptor id and length for later */ + txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].id = + desc_current; + txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].len = 0; + vhost_user_log_dirty_ring (vui, txvq, + ring[txvq->last_used_idx & txvq->qsz_mask]); + + /* The buffer should already be initialized */ + b_head->total_length_not_including_first_buffer = 0; + b_head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; + + if (PREDICT_FALSE (n_trace)) { - vlib_buffer_t *b_head, *b_current; - u32 bi_current; - u16 desc_current; - u32 desc_data_offset; - vring_desc_t *desc_table = txvq->desc; + //TODO: next_index is not exactly known at that point + vlib_trace_buffer (vm, node, next_index, b_head, + /* follow_chain */ 0); + vhost_trace_t *t0 = + vlib_add_trace (vm, node, b_head, sizeof (t0[0])); + vhost_user_rx_trace (t0, vui, qid, b_head, txvq); + n_trace--; + vlib_set_trace_count (vm, node, n_trace); + } - if (PREDICT_FALSE (cpu->rx_buffers_len <= 1)) + /* This depends on the setup but is very consistent + * So I think the CPU branch predictor will make a pretty good job + * at optimizing the decision. */ + if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT) + { + desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr, + &map_hint); + desc_current = 0; + if (PREDICT_FALSE (desc_table == 0)) { - /* Not enough rx_buffers - * Note: We yeld on 1 so we don't need to do an additional - * check for the next buffer prefetch. - */ - n_left = 0; - break; + vlib_error_count (vm, node->node_index, + VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1); + goto out; } + } - desc_current = - txvq->avail->ring[txvq->last_avail_idx & txvq->qsz_mask]; - cpu->rx_buffers_len--; - bi_current = cpu->rx_buffers[cpu->rx_buffers_len]; - b_head = b_current = vlib_get_buffer (vm, bi_current); - to_next[0] = bi_current; //We do that now so we can forget about bi_current - to_next++; - n_left_to_next--; - - vlib_prefetch_buffer_with_index - (vm, cpu->rx_buffers[cpu->rx_buffers_len - 1], LOAD); - - /* Just preset the used descriptor id and length for later */ - txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].id = - desc_current; - txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].len = 0; - vhost_user_log_dirty_ring (vui, txvq, - ring[txvq->last_used_idx & - txvq->qsz_mask]); - - /* The buffer should already be initialized */ - b_head->total_length_not_including_first_buffer = 0; - b_head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; - - if (PREDICT_FALSE (n_trace)) - { - //TODO: next_index is not exactly known at that point - vlib_trace_buffer (vm, node, next_index, b_head, - /* follow_chain */ 0); - vhost_trace_t *t0 = - vlib_add_trace (vm, node, b_head, sizeof (t0[0])); - vhost_user_rx_trace (t0, vui, qid, b_head, txvq); - n_trace--; - vlib_set_trace_count (vm, node, n_trace); - } + if (PREDICT_TRUE (vui->is_any_layout) || + (!(desc_table[desc_current].flags & VIRTQ_DESC_F_NEXT))) + { + /* ANYLAYOUT or single buffer */ + desc_data_offset = vui->virtio_net_hdr_sz; + } + else + { + /* CSR case without ANYLAYOUT, skip 1st buffer */ + desc_data_offset = desc_table[desc_current].len; + } - /* This depends on the setup but is very consistent - * So I think the CPU branch predictor will make a pretty good job - * at optimizing the decision. */ - if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT) + while (1) + { + /* Get more input if necessary. Or end of packet. */ + if (desc_data_offset == desc_table[desc_current].len) { - desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr, - &map_hint); - desc_current = 0; - if (PREDICT_FALSE (desc_table == 0)) + if (PREDICT_FALSE (desc_table[desc_current].flags & + VIRTQ_DESC_F_NEXT)) + { + desc_current = desc_table[desc_current].next; + desc_data_offset = 0; + } + else { - vlib_error_count (vm, node->node_index, - VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1); goto out; } } - if (PREDICT_TRUE (vui->is_any_layout) || - (!(desc_table[desc_current].flags & VIRTQ_DESC_F_NEXT))) - { - /* ANYLAYOUT or single buffer */ - desc_data_offset = vui->virtio_net_hdr_sz; - } - else - { - /* CSR case without ANYLAYOUT, skip 1st buffer */ - desc_data_offset = desc_table[desc_current].len; - } - - while (1) + /* Get more output if necessary. Or end of packet. */ + if (PREDICT_FALSE + (b_current->current_length == VLIB_BUFFER_DATA_SIZE)) { - /* Get more input if necessary. Or end of packet. */ - if (desc_data_offset == desc_table[desc_current].len) - { - if (PREDICT_FALSE (desc_table[desc_current].flags & - VIRTQ_DESC_F_NEXT)) - { - desc_current = desc_table[desc_current].next; - desc_data_offset = 0; - } - else - { - goto out; - } - } - - /* Get more output if necessary. Or end of packet. */ - if (PREDICT_FALSE - (b_current->current_length == VLIB_BUFFER_DATA_SIZE)) + if (PREDICT_FALSE (cpu->rx_buffers_len == 0)) { - if (PREDICT_FALSE (cpu->rx_buffers_len == 0)) - { - /* Cancel speculation */ - to_next--; - n_left_to_next++; - - /* - * Checking if there are some left buffers. - * If not, just rewind the used buffers and stop. - * Note: Scheduled copies are not cancelled. This is - * not an issue as they would still be valid. Useless, - * but valid. - */ - vhost_user_input_rewind_buffers (vm, cpu, b_head); - n_left = 0; - goto stop; - } - - /* Get next output */ - cpu->rx_buffers_len--; - u32 bi_next = cpu->rx_buffers[cpu->rx_buffers_len]; - b_current->next_buffer = bi_next; - b_current->flags |= VLIB_BUFFER_NEXT_PRESENT; - bi_current = bi_next; - b_current = vlib_get_buffer (vm, bi_current); + /* Cancel speculation */ + to_next--; + n_left_to_next++; + + /* + * Checking if there are some left buffers. + * If not, just rewind the used buffers and stop. + * Note: Scheduled copies are not cancelled. This is + * not an issue as they would still be valid. Useless, + * but valid. + */ + vhost_user_input_rewind_buffers (vm, cpu, b_head); + n_left = 0; + goto stop; } - /* Prepare a copy order executed later for the data */ - vhost_copy_t *cpy = &cpu->copy[copy_len]; - copy_len++; - u32 desc_data_l = - desc_table[desc_current].len - desc_data_offset; - cpy->len = VLIB_BUFFER_DATA_SIZE - b_current->current_length; - cpy->len = (cpy->len > desc_data_l) ? desc_data_l : cpy->len; - cpy->dst = (uword) (vlib_buffer_get_current (b_current) + - b_current->current_length); - cpy->src = desc_table[desc_current].addr + desc_data_offset; - - desc_data_offset += cpy->len; - - b_current->current_length += cpy->len; - b_head->total_length_not_including_first_buffer += cpy->len; + /* Get next output */ + cpu->rx_buffers_len--; + u32 bi_next = cpu->rx_buffers[cpu->rx_buffers_len]; + b_current->next_buffer = bi_next; + b_current->flags |= VLIB_BUFFER_NEXT_PRESENT; + bi_current = bi_next; + b_current = vlib_get_buffer (vm, bi_current); } - out: - CLIB_PREFETCH (&n_left, sizeof (n_left), LOAD); + /* Prepare a copy order executed later for the data */ + vhost_copy_t *cpy = &cpu->copy[copy_len]; + copy_len++; + u32 desc_data_l = desc_table[desc_current].len - desc_data_offset; + cpy->len = VLIB_BUFFER_DATA_SIZE - b_current->current_length; + cpy->len = (cpy->len > desc_data_l) ? desc_data_l : cpy->len; + cpy->dst = (uword) (vlib_buffer_get_current (b_current) + + b_current->current_length); + cpy->src = desc_table[desc_current].addr + desc_data_offset; - n_rx_bytes += b_head->total_length_not_including_first_buffer; - n_rx_packets++; + desc_data_offset += cpy->len; - b_head->total_length_not_including_first_buffer -= - b_head->current_length; + b_current->current_length += cpy->len; + b_head->total_length_not_including_first_buffer += cpy->len; + } - /* consume the descriptor and return it as used */ - txvq->last_avail_idx++; - txvq->last_used_idx++; + out: - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b_head); + n_rx_bytes += b_head->total_length_not_including_first_buffer; + n_rx_packets++; - vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index; - vnet_buffer (b_head)->sw_if_index[VLIB_TX] = (u32) ~ 0; - b_head->error = 0; + b_head->total_length_not_including_first_buffer -= + b_head->current_length; - if (current_config_index != ~(u32) 0) - { - b_head->current_config_index = current_config_index; - vnet_buffer (b_head)->feature_arc_index = feature_arc_idx; - } + /* consume the descriptor and return it as used */ + txvq->last_avail_idx++; + txvq->last_used_idx++; - n_left--; + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b_head); - /* - * Although separating memory copies from virtio ring parsing - * is beneficial, we can offer to perform the copies from time - * to time in order to free some space in the ring. - */ - if (PREDICT_FALSE (copy_len >= VHOST_USER_RX_COPY_THRESHOLD)) - { - if (PREDICT_FALSE (vhost_user_input_copy (vui, cpu->copy, - copy_len, &map_hint))) - { - vlib_error_count (vm, node->node_index, - VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1); - } - copy_len = 0; + vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index; + vnet_buffer (b_head)->sw_if_index[VLIB_TX] = (u32) ~ 0; + b_head->error = 0; + + if (current_config_index != ~(u32) 0) + { + b_head->current_config_index = current_config_index; + vnet_buffer (b_head)->feature_arc_index = feature_arc_idx; + } - /* give buffers back to driver */ - CLIB_MEMORY_BARRIER (); - txvq->used->idx = txvq->last_used_idx; - vhost_user_log_dirty_ring (vui, txvq, idx); + n_left--; + + /* + * Although separating memory copies from virtio ring parsing + * is beneficial, we can offer to perform the copies from time + * to time in order to free some space in the ring. + */ + if (PREDICT_FALSE (copy_len >= VHOST_USER_RX_COPY_THRESHOLD)) + { + if (PREDICT_FALSE (vhost_user_input_copy (vui, cpu->copy, + copy_len, &map_hint))) + { + vlib_error_count (vm, node->node_index, + VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1); } + copy_len = 0; + + /* give buffers back to driver */ + CLIB_MEMORY_BARRIER (); + txvq->used->idx = txvq->last_used_idx; + vhost_user_log_dirty_ring (vui, txvq, idx); } - stop: - vlib_put_next_frame (vm, node, next_index, n_left_to_next); } +stop: + vlib_put_next_frame (vm, node, next_index, n_left_to_next); /* Do the memory copies */ if (PREDICT_FALSE (vhost_user_input_copy (vui, cpu->copy, copy_len, -- 2.16.6