X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fpg%2Finput.c;h=ee6aad4995e38ad4df6475c893f5f96b7c41aae8;hb=775f73c;hp=f31152c3889abe1fce770591dfa59a454f1c59ad;hpb=bd846cdc5d99260274a02e9bb474211ef32d031c;p=vpp.git diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c index f31152c3889..ee6aad4995e 100644 --- a/src/vnet/pg/input.c +++ b/src/vnet/pg/input.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -1083,6 +1084,7 @@ init_replay_buffers_inline (vlib_main_t * vm, vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; d0 = vec_elt (s->replay_packet_templates, i); + vnet_buffer2 (b0)->pg_replay_timestamp = s->replay_packet_timestamps[i]; n0 = n_data; if (data_offset + n_data >= vec_len (d0)) @@ -1090,7 +1092,7 @@ init_replay_buffers_inline (vlib_main_t * vm, b0->current_length = n0; - clib_memcpy (b0->data, d0 + data_offset, n0); + clib_memcpy_fast (b0->data, d0 + data_offset, n0); i = i + 1 == l ? 0 : i + 1; } } @@ -1147,8 +1149,8 @@ init_buffers_inline (vlib_main_t * vm, if (set_data) { - clib_memcpy (b0->data, data, n_data); - clib_memcpy (b1->data, data, n_data); + clib_memcpy_fast (b0->data, data, n_data); + clib_memcpy_fast (b1->data, data, n_data); } else { @@ -1172,31 +1174,12 @@ init_buffers_inline (vlib_main_t * vm, vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; if (set_data) - clib_memcpy (b0->data, data, n_data); + clib_memcpy_fast (b0->data, data, n_data); else ASSERT (validate_buffer_data2 (b0, s, data_offset, n_data)); } } -static void -pg_buffer_init (vlib_main_t * vm, - vlib_buffer_free_list_t * fl, u32 * buffers, u32 n_buffers) -{ - pg_main_t *pg = &pg_main; - pg_stream_t *s; - uword bi, si; - - si = fl->buffer_init_function_opaque & pow2_mask (24); - bi = fl->buffer_init_function_opaque >> 24; - - s = pool_elt_at_index (pg->streams, si); - - init_buffers_inline (vm, s, buffers, n_buffers, - /* data_offset */ bi * s->buffer_bytes, - /* n_data */ s->buffer_bytes, - /* set_data */ 1); -} - static u32 pg_stream_fill_helper (pg_main_t * pg, pg_stream_t * s, @@ -1204,26 +1187,9 @@ pg_stream_fill_helper (pg_main_t * pg, u32 * buffers, u32 * next_buffers, u32 n_alloc) { vlib_main_t *vm = vlib_get_main (); - vlib_buffer_free_list_t *f; uword is_start_of_packet = bi == s->buffer_indices; u32 n_allocated; - f = vlib_buffer_get_free_list (vm, bi->free_list_index); - - /* - * Historically, the pg maintained its own free lists and - * device drivers tx paths would return pkts. - */ - if (vm->buffer_main->callbacks_registered == 0 && - !(s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE)) - f->buffer_init_function = pg_buffer_init; - f->buffer_init_function_opaque = - (s - pg->streams) | ((bi - s->buffer_indices) << 24); - - if (is_start_of_packet) - vnet_buffer (&f->buffer_init_template)->sw_if_index[VLIB_RX] - = vnet_main.local_interface_sw_if_index; - n_allocated = vlib_buffer_alloc_from_free_list (vm, buffers, n_alloc, @@ -1238,16 +1204,12 @@ pg_stream_fill_helper (pg_main_t * pg, n_alloc = n_allocated; /* Reinitialize buffers */ - if (vm->buffer_main->callbacks_registered == 0 || CLIB_DEBUG > 0 - || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE)) - init_buffers_inline - (vm, s, - buffers, - n_alloc, (bi - s->buffer_indices) * s->buffer_bytes /* data offset */ , - s->buffer_bytes, - /* set_data */ - vm->buffer_main->callbacks_registered != 0 - || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE) != 0); + init_buffers_inline + (vm, s, + buffers, + n_alloc, (bi - s->buffer_indices) * s->buffer_bytes /* data offset */ , + s->buffer_bytes, + /* set_data */ 1); if (next_buffers) pg_set_next_buffer_pointers (pg, s, buffers, next_buffers, n_alloc); @@ -1463,13 +1425,15 @@ pg_input_trace (pg_main_t * pg, t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX]; - clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data)); - clib_memcpy (&t1->buffer, b1, sizeof (b1[0]) - sizeof (b1->pre_data)); + clib_memcpy_fast (&t0->buffer, b0, + sizeof (b0[0]) - sizeof (b0->pre_data)); + clib_memcpy_fast (&t1->buffer, b1, + sizeof (b1[0]) - sizeof (b1->pre_data)); - clib_memcpy (t0->buffer.pre_data, b0->data, - sizeof (t0->buffer.pre_data)); - clib_memcpy (t1->buffer.pre_data, b1->data, - sizeof (t1->buffer.pre_data)); + clib_memcpy_fast (t0->buffer.pre_data, b0->data, + sizeof (t0->buffer.pre_data)); + clib_memcpy_fast (t1->buffer.pre_data, b1->data, + sizeof (t1->buffer.pre_data)); } while (n_left >= 1) @@ -1490,9 +1454,10 @@ pg_input_trace (pg_main_t * pg, t0->stream_index = stream_index; t0->packet_length = vlib_buffer_length_in_chain (vm, b0); t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data)); - clib_memcpy (t0->buffer.pre_data, b0->data, - sizeof (t0->buffer.pre_data)); + clib_memcpy_fast (&t0->buffer, b0, + sizeof (b0[0]) - sizeof (b0->pre_data)); + clib_memcpy_fast (t0->buffer.pre_data, b0->data, + sizeof (t0->buffer.pre_data)); } } @@ -1532,7 +1497,24 @@ pg_generate_packets (vlib_node_runtime_t * node, { u32 *head, *start, *end; - vlib_get_next_frame (vm, node, next_index, to_next, n_left); + if (PREDICT_TRUE (next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)) + { + vlib_next_frame_t *nf; + vlib_frame_t *f; + ethernet_input_frame_t *ef; + pg_interface_t *pi; + vlib_get_new_next_frame (vm, node, next_index, to_next, n_left); + nf = vlib_node_runtime_get_next_frame (vm, node, next_index); + f = vlib_get_frame (vm, nf->frame_index); + f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX; + + ef = vlib_frame_scalar_args (f); + pi = pool_elt_at_index (pg->interfaces, s->pg_if_index); + ef->sw_if_index = pi->sw_if_index; + ef->hw_if_index = pi->hw_if_index; + } + else + vlib_get_next_frame (vm, node, next_index, to_next, n_left); n_this_frame = n_packets_to_generate; if (n_this_frame > n_left) @@ -1559,11 +1541,8 @@ pg_generate_packets (vlib_node_runtime_t * node, { vlib_buffer_t *b; b = vlib_get_buffer (vm, to_next[i]); - vnet_buffer (b)->device_input_feat.saved_next_index = - s->next_index; - vnet_buffer (b)->device_input_feat.buffer_advance = 0; b->current_config_index = current_config_index; - b->feature_arc_index = feature_arc_index; + vnet_buffer (b)->feature_arc_index = feature_arc_index; } n_trace = vlib_get_trace_count (vm, node);