X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fmemif%2Fnode.c;h=aef8ffbd0c2ca78f2e3003fd8fd70dfd90d6481f;hb=e10d167bfe8aec76cee138cbe33166ba89423c8c;hp=8bd9fcddcb60448a65b5f4050a25df618c7d1eb6;hpb=178cf493d009995b28fdf220f04c98860ff79a9b;p=vpp.git diff --git a/src/plugins/memif/node.c b/src/plugins/memif/node.c index 8bd9fcddcb6..aef8ffbd0c2 100644 --- a/src/plugins/memif/node.c +++ b/src/plugins/memif/node.c @@ -180,15 +180,18 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, memif_main_t *mm = &memif_main; memif_ring_t *ring; memif_queue_t *mq; - u16 buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES; + u16 buffer_size = vlib_buffer_get_default_data_size (vm); uword n_trace = vlib_get_trace_count (vm, node); + u16 nexts[MEMIF_RX_VECTOR_SZ], *next = nexts; + u32 _to_next_bufs[MEMIF_RX_VECTOR_SZ], *to_next_bufs = _to_next_bufs, *bi; u32 n_rx_packets = 0, n_rx_bytes = 0; - u32 n_left; + u32 n_left, n_left_to_next; + u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; vlib_buffer_t *b0, *b1, *b2, *b3; u32 thread_index = vm->thread_index; memif_per_thread_data_t *ptd = vec_elt_at_index (mm->per_thread_data, thread_index); - vlib_buffer_t *bt = &ptd->buffer_template; + vlib_buffer_t bt; u16 cur_slot, last_slot, ring_size, n_slots, mask; i16 start_offset; u16 n_buffers = 0, n_alloc; @@ -278,7 +281,8 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, /* allocate free buffers */ vec_validate_aligned (ptd->buffers, n_buffers - 1, CLIB_CACHE_LINE_BYTES); - n_alloc = vlib_buffer_alloc (vm, ptd->buffers, n_buffers); + n_alloc = vlib_buffer_alloc_from_pool (vm, ptd->buffers, n_buffers, + mq->buffer_pool_index); if (PREDICT_FALSE (n_alloc != n_buffers)) { if (n_alloc) @@ -335,36 +339,57 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, mq->last_tail = cur_slot; } - u16 nexts[MEMIF_RX_VECTOR_SZ], *next = nexts; - u32 to_next_buffers[MEMIF_RX_VECTOR_SZ], *bi = to_next_buffers; - /* prepare buffer template and next indices */ - vnet_buffer (bt)->sw_if_index[VLIB_RX] = mif->sw_if_index; - vnet_buffer (bt)->feature_arc_index = 0; - bt->current_data = start_offset; - bt->current_config_index = 0; + vnet_buffer (&ptd->buffer_template)->sw_if_index[VLIB_RX] = + mif->sw_if_index; + vnet_buffer (&ptd->buffer_template)->feature_arc_index = 0; + ptd->buffer_template.current_data = start_offset; + ptd->buffer_template.current_config_index = 0; + ptd->buffer_template.buffer_pool_index = mq->buffer_pool_index; + ptd->buffer_template.ref_count = 1; if (mode == MEMIF_INTERFACE_MODE_ETHERNET) { - u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; + next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; if (mif->per_interface_next_index != ~0) next_index = mif->per_interface_next_index; else vnet_feature_start_device_input_x1 (mif->sw_if_index, &next_index, - bt); - clib_memset_u16 (nexts, next_index, n_rx_packets); + &ptd->buffer_template); + + vlib_get_new_next_frame (vm, node, next_index, to_next_bufs, + n_left_to_next); + if (PREDICT_TRUE (next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)) + { + vlib_next_frame_t *nf; + vlib_frame_t *f; + ethernet_input_frame_t *ef; + nf = vlib_node_runtime_get_next_frame (vm, node, next_index); + f = vlib_get_frame (vm, nf->frame); + f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX; + + ef = vlib_frame_scalar_args (f); + ef->sw_if_index = mif->sw_if_index; + ef->hw_if_index = mif->hw_if_index; + vlib_frame_no_append (f); + } } /* process buffer metadata */ u32 n_from = n_rx_packets; po = ptd->packet_ops; + bi = to_next_bufs; + + /* copy template into local variable - will save per packet load */ + vlib_buffer_copy_template (&bt, &ptd->buffer_template); while (n_from >= 8) { - b0 = vlib_get_buffer (vm, po[4].first_buffer_vec_index); - b1 = vlib_get_buffer (vm, po[5].first_buffer_vec_index); - b2 = vlib_get_buffer (vm, po[6].first_buffer_vec_index); - b3 = vlib_get_buffer (vm, po[7].first_buffer_vec_index); + b0 = vlib_get_buffer (vm, ptd->buffers[po[0].first_buffer_vec_index]); + b1 = vlib_get_buffer (vm, ptd->buffers[po[1].first_buffer_vec_index]); + b2 = vlib_get_buffer (vm, ptd->buffers[po[2].first_buffer_vec_index]); + b3 = vlib_get_buffer (vm, ptd->buffers[po[3].first_buffer_vec_index]); + vlib_prefetch_buffer_header (b0, STORE); vlib_prefetch_buffer_header (b1, STORE); vlib_prefetch_buffer_header (b2, STORE); @@ -387,7 +412,10 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, b2 = vlib_get_buffer (vm, bi[2]); b3 = vlib_get_buffer (vm, bi[3]); - clib_memcpy64_x4 (b0, b1, b2, b3, bt); + vlib_buffer_copy_template (b0, &bt); + vlib_buffer_copy_template (b1, &bt); + vlib_buffer_copy_template (b2, &bt); + vlib_buffer_copy_template (b3, &bt); b0->current_length = po[0].packet_len; n_rx_bytes += b0->current_length; @@ -424,7 +452,7 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, fbvi[0] = po[0].first_buffer_vec_index; bi[0] = ptd->buffers[fbvi[0]]; b0 = vlib_get_buffer (vm, bi[0]); - clib_memcpy_fast (b0, bt, 64); + vlib_buffer_copy_template (b0, &bt); b0->current_length = po->packet_len; n_rx_bytes += b0->current_length; @@ -446,16 +474,19 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node)))) { u32 n_left = n_rx_packets; - bi = to_next_buffers; + bi = to_next_bufs; next = nexts; + u32 ni = next_index; while (n_trace && n_left) { vlib_buffer_t *b; memif_input_trace_t *tr; + if (mode != MEMIF_INTERFACE_MODE_ETHERNET) + ni = next[0]; b = vlib_get_buffer (vm, bi[0]); - vlib_trace_buffer (vm, node, next[0], b, /* follow_chain */ 0); + vlib_trace_buffer (vm, node, ni, b, /* follow_chain */ 0); tr = vlib_add_trace (vm, node, b, sizeof (*tr)); - tr->next_index = next[0]; + tr->next_index = ni; tr->hw_if_index = mif->hw_if_index; tr->ring = qid; @@ -468,8 +499,13 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_set_trace_count (vm, node, n_trace); } - vlib_buffer_enqueue_to_next (vm, node, to_next_buffers, nexts, - n_rx_packets); + if (mode == MEMIF_INTERFACE_MODE_ETHERNET) + { + n_left_to_next -= n_rx_packets; + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + else + vlib_buffer_enqueue_to_next (vm, node, to_next_bufs, nexts, n_rx_packets); vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thread_index, @@ -536,7 +572,7 @@ memif_device_input_zc_inline (vlib_main_t * vm, vlib_node_runtime_t * node, /* asume that somebody will want to add ethernet header on the packet so start with IP header at offset 14 */ start_offset = (mode == MEMIF_INTERFACE_MODE_IP) ? 14 : 0; - buffer_length = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES - start_offset; + buffer_length = vlib_buffer_get_default_data_size (vm) - start_offset; cur_slot = mq->last_tail; last_slot = ring->tail; @@ -753,8 +789,9 @@ refill: clib_memset (dt, 0, sizeof (memif_desc_t)); dt->length = buffer_length; - n_alloc = vlib_buffer_alloc_to_ring (vm, mq->buffers, head & mask, - ring_size, n_slots); + n_alloc = vlib_buffer_alloc_to_ring_from_pool (vm, mq->buffers, head & mask, + ring_size, n_slots, + mq->buffer_pool_index); if (PREDICT_FALSE (n_alloc != n_slots)) { @@ -888,6 +925,7 @@ VLIB_NODE_FN (memif_input_node) (vlib_main_t * vm, /* *INDENT-OFF* */ VLIB_REGISTER_NODE (memif_input_node) = { .name = "memif-input", + .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED, .sibling_of = "device-input", .format_trace = format_memif_input_trace, .type = VLIB_NODE_TYPE_INPUT,