X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Finterface_output.c;h=659273b53cd4e7036bd180c0987d2746242f65b6;hb=0d05c0d214ffd326e531bea58f3c971bb9a58252;hp=72ceb95132d4786a0ed03e2735b0b17fcd81b75e;hpb=5d5f85f5e4003476fb6d9a0ccd6ad58ad90e5138;p=vpp.git diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c index 72ceb95132d..659273b53cd 100644 --- a/src/vnet/interface_output.c +++ b/src/vnet/interface_output.c @@ -46,10 +46,12 @@ #include #include #include +#include #include #include #include #include +#include typedef struct { @@ -176,8 +178,9 @@ vnet_interface_output_handle_offload (vlib_main_t *vm, vlib_buffer_t *b) static_always_inline uword vnet_interface_output_node_inline (vlib_main_t *vm, u32 sw_if_index, vlib_combined_counter_main_t *ccm, - vlib_buffer_t **b, u32 config_index, u8 arc, - u32 n_left, int processing_level) + vlib_buffer_t **b, void **p, + u32 config_index, u8 arc, u32 n_left, + int processing_level) { u32 n_bytes = 0; u32 n_bytes0, n_bytes1, n_bytes2, n_bytes3; @@ -208,6 +211,15 @@ vnet_interface_output_node_inline (vlib_main_t *vm, u32 sw_if_index, n_bytes += n_bytes2 = vlib_buffer_length_in_chain (vm, b[2]); n_bytes += n_bytes3 = vlib_buffer_length_in_chain (vm, b[3]); + if (processing_level >= 3) + { + p[0] = vlib_buffer_get_current (b[0]); + p[1] = vlib_buffer_get_current (b[1]); + p[2] = vlib_buffer_get_current (b[2]); + p[3] = vlib_buffer_get_current (b[3]); + p += 4; + } + if (processing_level >= 2) { u32 tx_swif0, tx_swif1, tx_swif2, tx_swif3; @@ -262,6 +274,12 @@ vnet_interface_output_node_inline (vlib_main_t *vm, u32 sw_if_index, n_bytes += n_bytes0 = vlib_buffer_length_in_chain (vm, b[0]); + if (processing_level >= 3) + { + p[0] = vlib_buffer_get_current (b[0]); + p += 1; + } + if (processing_level >= 2) { u32 tx_swif0 = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; @@ -344,39 +362,71 @@ vnet_interface_pcap_tx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, } static_always_inline void -store_tx_frame_scalar_data (vnet_hw_if_output_node_runtime_t *r, - vnet_hw_if_tx_frame_t *tf) +hash_func_with_mask (void **p, u32 *hash, u32 n_packets, u32 *lookup_table, + u32 mask, vnet_hash_fn_t hf) { - if (r) - clib_memcpy_fast (tf, &r->frame, sizeof (vnet_hw_if_tx_frame_t)); + u32 n_left_from = n_packets; + + hf (p, hash, n_packets); + + clib_array_mask_u32 (hash, mask, n_packets); + + while (n_left_from >= 4) + { + hash[0] = lookup_table[hash[0]]; + hash[1] = lookup_table[hash[1]]; + hash[2] = lookup_table[hash[2]]; + hash[3] = lookup_table[hash[3]]; + + hash += 4; + n_left_from -= 4; + } + + while (n_left_from > 0) + { + hash[0] = lookup_table[hash[0]]; + + hash += 1; + n_left_from -= 1; + } } static_always_inline void -enqueu_to_tx_node (vlib_main_t *vm, vlib_node_runtime_t *node, - vnet_hw_interface_t *hi, u32 *from, u32 n_vectors) +store_tx_frame_scalar_data (vnet_hw_if_tx_frame_t *copy_frame, + vnet_hw_if_tx_frame_t *tf) { - u32 next_index = VNET_INTERFACE_OUTPUT_NEXT_TX; - vnet_hw_if_output_node_runtime_t *r = 0; - u32 n_free, n_copy, *to; - vnet_hw_if_tx_frame_t *tf; - vlib_frame_t *f; - - ASSERT (n_vectors <= VLIB_FRAME_SIZE); + if (copy_frame) + clib_memcpy_fast (tf, copy_frame, sizeof (vnet_hw_if_tx_frame_t)); +} - if (hi->output_node_thread_runtimes) - r = vec_elt_at_index (hi->output_node_thread_runtimes, vm->thread_index); +static_always_inline u32 +enqueue_one_to_tx_node (vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ppqi, + u32 *from, vnet_hw_if_tx_frame_t *copy_frame, + u32 n_vectors, u32 n_left, u32 next_index) +{ + u32 tmp[VLIB_FRAME_SIZE]; + u64 mask[VLIB_FRAME_SIZE / 64] = {}; + vlib_frame_t *f; + vnet_hw_if_tx_frame_t *tf; + u32 *to; + u32 n_copy = 0, n_free = 0; f = vlib_get_next_frame_internal (vm, node, next_index, 0); tf = vlib_frame_scalar_args (f); - if (f->n_vectors > 0 && (r == 0 || tf->queue_id == r->frame.queue_id)) + if (f->n_vectors > 0 && + (!copy_frame || (tf->queue_id == copy_frame->queue_id))) { /* append current next frame */ n_free = VLIB_FRAME_SIZE - f->n_vectors; - n_copy = clib_min (n_vectors, n_free); - n_vectors -= n_copy; - to = vlib_frame_vector_args (f); - to += f->n_vectors; + /* + * if frame contains enough space for worst case scenario, + * we can avoid use of tmp + */ + if (n_free >= n_left) + to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors; + else + to = tmp; } else { @@ -388,25 +438,113 @@ enqueu_to_tx_node (vlib_main_t *vm, vlib_node_runtime_t *node, } /* empty frame - store scalar data */ - store_tx_frame_scalar_data (r, tf); + store_tx_frame_scalar_data (copy_frame, tf); to = vlib_frame_vector_args (f); n_free = VLIB_FRAME_SIZE; - n_copy = n_vectors; - n_vectors = 0; } - vlib_buffer_copy_indices (to, from, n_copy); - vlib_put_next_frame (vm, node, next_index, n_free - n_copy); + /* + * per packet queue id array + * compare with given queue_id, if match, copy respective buffer index from + * -> to + */ + if (ppqi) + { + clib_mask_compare_u32 (copy_frame->queue_id, ppqi, mask, n_vectors); + n_copy = clib_compress_u32 (to, from, mask, n_vectors); - if (n_vectors == 0) - return; + if (n_copy == 0) + return n_left; + } + else + { + /* + * no work required, just copy all buffer indices from -> to + */ + n_copy = n_left; + vlib_buffer_copy_indices (to, from, n_copy); + } + + if (to != tmp) + { + /* indices already written to frame, just close it */ + vlib_put_next_frame (vm, node, next_index, n_free - n_copy); + } + else if (n_free >= n_copy) + { + /* enough space in the existing frame */ + to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors; + vlib_buffer_copy_indices (to, tmp, n_copy); + vlib_put_next_frame (vm, node, next_index, n_free - n_copy); + } + else + { + /* full frame */ + to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors; + vlib_buffer_copy_indices (to, tmp, n_free); + vlib_put_next_frame (vm, node, next_index, 0); + + /* second frame */ + u32 n_2nd_frame = n_copy - n_free; + f = vlib_get_next_frame_internal (vm, node, next_index, 1); + tf = vlib_frame_scalar_args (f); + /* empty frame - store scalar data */ + store_tx_frame_scalar_data (copy_frame, tf); + to = vlib_frame_vector_args (f); + vlib_buffer_copy_indices (to, tmp + n_free, n_2nd_frame); + vlib_put_next_frame (vm, node, next_index, + VLIB_FRAME_SIZE - n_2nd_frame); + } + + return n_left - n_copy; +} + +static_always_inline void +enqueue_to_tx_node (vlib_main_t *vm, vlib_node_runtime_t *node, + vnet_hw_interface_t *hi, u32 next_index, + vnet_hw_if_output_node_runtime_t *r, u32 *from, void **p, + u32 n_vectors) +{ + u32 n_left = n_vectors; + + ASSERT (n_vectors <= VLIB_FRAME_SIZE); + + /* + * backward compatible for drivers not integrated with new tx infra. + */ + if (r == 0) + { + n_left = enqueue_one_to_tx_node (vm, node, NULL, from, NULL, n_vectors, + n_left, next_index); + } + /* + * only 1 tx queue of given interface is available on given thread + */ + else if (r->n_queues == 1) + { + n_left = enqueue_one_to_tx_node (vm, node, NULL, from, r->frame, + n_vectors, n_left, next_index); + } + /* + * multi tx-queues use case + */ + else if (r->n_queues > 1) + { + u32 qids[VLIB_FRAME_SIZE]; + + hash_func_with_mask (p, qids, n_vectors, r->lookup_table, + vec_len (r->lookup_table) - 1, hi->hf); - /* we have more indices to store, take empty frame */ - from += n_copy; - f = vlib_get_next_frame_internal (vm, node, next_index, 1); - store_tx_frame_scalar_data (r, vlib_frame_scalar_args (f)); - vlib_buffer_copy_indices (vlib_frame_vector_args (f), from, n_vectors); - vlib_put_next_frame (vm, node, next_index, VLIB_FRAME_SIZE - n_vectors); + for (u32 i = 0; i < r->n_queues; i++) + { + n_left = enqueue_one_to_tx_node (vm, node, qids, from, &r->frame[i], + n_vectors, n_left, next_index); + if (n_left == 0) + break; + } + } + else + ASSERT (0); } VLIB_NODE_FN (vnet_interface_output_node) @@ -418,6 +556,7 @@ VLIB_NODE_FN (vnet_interface_output_node) vnet_hw_interface_t *hi; vnet_sw_interface_t *si; vnet_interface_output_runtime_t *rt = (void *) node->runtime_data; + vnet_hw_if_output_node_runtime_t *r = 0; vlib_buffer_t *bufs[VLIB_FRAME_SIZE]; u32 n_bytes, n_buffers = frame->n_vectors; u32 config_index = ~0; @@ -427,6 +566,8 @@ VLIB_NODE_FN (vnet_interface_output_node) u8 arc = im->output_feature_arc_index; int arc_or_subif = 0; int do_tx_offloads = 0; + void *ptr[VLIB_FRAME_SIZE], **p = ptr; + u8 is_parr = 0; u32 *from; if (node->flags & VLIB_NODE_FLAG_TRACE) @@ -462,6 +603,27 @@ VLIB_NODE_FN (vnet_interface_output_node) node->node_index, VNET_INTERFACE_OUTPUT_ERROR_INTERFACE_DOWN); } + if (hi->output_node_thread_runtimes) + r = vec_elt_at_index (hi->output_node_thread_runtimes, vm->thread_index); + + if (r) + { + /* + * tx queue of given interface is not available on given thread + */ + if (r->n_queues == 0) + return vlib_error_drop_buffers ( + vm, node, from, + /* buffer stride */ 1, n_buffers, VNET_INTERFACE_OUTPUT_NEXT_DROP, + node->node_index, VNET_INTERFACE_OUTPUT_ERROR_NO_TX_QUEUE); + /* + * multiple tx queues available on given thread + */ + else if (r->n_queues > 1) + /* construct array of pointer */ + is_parr = 1; + } + /* interface-output feature arc handling */ if (PREDICT_FALSE (vnet_have_features (arc, sw_if_index))) { @@ -482,20 +644,28 @@ VLIB_NODE_FN (vnet_interface_output_node) VNET_HW_INTERFACE_CAP_SUPPORTS_TX_CKSUM) do_tx_offloads = 1; - if (do_tx_offloads == 0 && arc_or_subif == 0) + // basic processing + if (do_tx_offloads == 0 && arc_or_subif == 0 && is_parr == 0) n_bytes = vnet_interface_output_node_inline ( - vm, sw_if_index, ccm, bufs, config_index, arc, n_buffers, 0); - else if (do_tx_offloads == 1 && arc_or_subif == 0) + vm, sw_if_index, ccm, bufs, NULL, config_index, arc, n_buffers, 0); + // basic processing + tx offloads + else if (do_tx_offloads == 1 && arc_or_subif == 0 && is_parr == 0) n_bytes = vnet_interface_output_node_inline ( - vm, sw_if_index, ccm, bufs, config_index, arc, n_buffers, 1); + vm, sw_if_index, ccm, bufs, NULL, config_index, arc, n_buffers, 1); + // basic processing + tx offloads + vlans + arcs + else if (do_tx_offloads == 1 && arc_or_subif == 1 && is_parr == 0) + n_bytes = vnet_interface_output_node_inline ( + vm, sw_if_index, ccm, bufs, NULL, config_index, arc, n_buffers, 2); + // basic processing + tx offloads + vlans + arcs + multi-txqs else n_bytes = vnet_interface_output_node_inline ( - vm, sw_if_index, ccm, bufs, config_index, arc, n_buffers, 2); + vm, sw_if_index, ccm, bufs, p, config_index, arc, n_buffers, 3); from = vlib_frame_vector_args (frame); if (PREDICT_TRUE (next_index == VNET_INTERFACE_OUTPUT_NEXT_TX)) { - enqueu_to_tx_node (vm, node, hi, from, frame->n_vectors); + enqueue_to_tx_node (vm, node, hi, next_index, r, from, ptr, + frame->n_vectors); } else { @@ -1087,16 +1257,14 @@ VLIB_NODE_FN (vnet_interface_output_arc_end_node) { vnet_main_t *vnm = vnet_get_main (); vnet_interface_main_t *im = &vnm->interface_main; - vnet_hw_if_output_node_runtime_t *r = 0; vnet_hw_interface_t *hi; - vnet_hw_if_tx_frame_t *tf; vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; u32 sw_if_indices[VLIB_FRAME_SIZE], *sw_if_index = sw_if_indices; u64 used_elts[VLIB_FRAME_SIZE / 64] = {}; u64 mask[VLIB_FRAME_SIZE / 64] = {}; - u32 *tmp, *from, n_left, n_free, n_comp, *to, swif, off; + u32 *tmp, *from, n_left, n_comp, n_p_comp, swif, off; u16 next_index; - vlib_frame_t *f; + void *ptr[VLIB_FRAME_SIZE], **p = ptr; from = vlib_frame_vector_args (frame); n_left = frame->n_vectors; @@ -1108,11 +1276,17 @@ VLIB_NODE_FN (vnet_interface_output_arc_end_node) vlib_prefetch_buffer_header (b[5], LOAD); vlib_prefetch_buffer_header (b[6], LOAD); vlib_prefetch_buffer_header (b[7], LOAD); + + p[0] = vlib_buffer_get_current (b[0]); + p[1] = vlib_buffer_get_current (b[1]); + p[2] = vlib_buffer_get_current (b[2]); + p[3] = vlib_buffer_get_current (b[3]); sw_if_index[0] = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; sw_if_index[1] = vnet_buffer (b[1])->sw_if_index[VLIB_TX]; sw_if_index[2] = vnet_buffer (b[2])->sw_if_index[VLIB_TX]; sw_if_index[3] = vnet_buffer (b[3])->sw_if_index[VLIB_TX]; + p += 4; b += 4; sw_if_index += 4; n_left -= 4; @@ -1120,7 +1294,9 @@ VLIB_NODE_FN (vnet_interface_output_arc_end_node) while (n_left) { + p[0] = vlib_buffer_get_current (b[0]); sw_if_index[0] = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; + p++; b++; sw_if_index++; n_left--; @@ -1137,68 +1313,40 @@ VLIB_NODE_FN (vnet_interface_output_arc_end_node) more: next_index = vec_elt (im->if_out_arc_end_next_index_by_sw_if_index, swif); hi = vnet_get_sup_hw_interface (vnm, swif); + vnet_hw_if_output_node_runtime_t *r = 0; + void *ptr_tmp[VLIB_FRAME_SIZE], **p_tmp = ptr_tmp; + if (hi->output_node_thread_runtimes) r = vec_elt_at_index (hi->output_node_thread_runtimes, vm->thread_index); - f = vlib_get_next_frame_internal (vm, node, next_index, 0); - tf = vlib_frame_scalar_args (f); - - if (f->n_vectors > 0 && (r == 0 || r->frame.queue_id == tf->queue_id)) - { - /* append frame */ - n_free = VLIB_FRAME_SIZE - f->n_vectors; - if (n_free >= f->n_vectors) - to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors; - else - to = tmp; - } - else - { - if (f->n_vectors > 0) - { - /* current frame doesn't fit - grab empty one */ - f = vlib_get_next_frame_internal (vm, node, next_index, 1); - tf = vlib_frame_scalar_args (f); - } - - /* empty frame - store scalar data */ - store_tx_frame_scalar_data (r, tf); - n_free = VLIB_FRAME_SIZE; - to = vlib_frame_vector_args (f); - } /* compare and compress based on comparison mask */ clib_mask_compare_u32 (swif, sw_if_indices, mask, frame->n_vectors); - n_comp = clib_compress_u32 (to, from, mask, frame->n_vectors); + n_comp = clib_compress_u32 (tmp, from, mask, frame->n_vectors); - if (tmp != to) - { - /* indices already written to frame, just close it */ - vlib_put_next_frame (vm, node, next_index, n_free - n_comp); - } - else if (n_free >= n_comp) + /* + * tx queue of given interface is not available on given thread + */ + if (r) { - /* enough space in the existing frame */ - to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors; - vlib_buffer_copy_indices (to, tmp, n_comp); - vlib_put_next_frame (vm, node, next_index, n_free - n_comp); + if (r->n_queues == 0) + { + vlib_error_drop_buffers ( + vm, node, tmp, + /* buffer stride */ 1, n_comp, VNET_INTERFACE_OUTPUT_NEXT_DROP, + node->node_index, VNET_INTERFACE_OUTPUT_ERROR_NO_TX_QUEUE); + goto drop; + } + else if (r->n_queues > 1) + { + n_p_comp = clib_compress_u64 ((u64 *) p_tmp, (u64 *) ptr, mask, + frame->n_vectors); + ASSERT (n_p_comp == n_comp); + } } - else - { - /* full frame */ - to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors; - vlib_buffer_copy_indices (to, tmp, n_free); - vlib_put_next_frame (vm, node, next_index, 0); - /* second frame */ - u32 n_frame2 = n_comp - n_free; - f = vlib_get_next_frame_internal (vm, node, next_index, 1); - to = vlib_frame_vector_args (f); - vlib_buffer_copy_indices (to, tmp + n_free, n_frame2); - tf = vlib_frame_scalar_args (f); - store_tx_frame_scalar_data (r, tf); - vlib_put_next_frame (vm, node, next_index, VLIB_FRAME_SIZE - n_frame2); - } + enqueue_to_tx_node (vm, node, hi, next_index, r, tmp, ptr_tmp, n_comp); +drop: n_left -= n_comp; if (n_left) {