X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fpg%2Finput.c;h=4db4b45edcb26644d83e0342cc207d89d4bd0651;hb=671e60e65635b8d030bf303c88411192c747b59e;hp=e15faeb85647fcb18b1b24c5b7166d10f2ccfce3;hpb=7cd468a3d7dee7d6c92f69a0bb7061ae208ec727;p=vpp.git diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c index e15faeb8564..4db4b45edcb 100644 --- a/src/vnet/pg/input.c +++ b/src/vnet/pg/input.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -65,7 +66,7 @@ validate_buffer_data2 (vlib_buffer_t * b, pg_stream_t * s, if (i >= n_bytes) return 1; - clib_warning ("buffer %U", format_vlib_buffer, b); + clib_warning ("buffer %U", format_vnet_buffer, b); clib_warning ("differ at index %d", i); clib_warning ("is %U", format_hex_bytes, bd, n_bytes); clib_warning ("mask %U", format_hex_bytes, pm, n_bytes); @@ -893,7 +894,7 @@ pg_generate_set_lengths (pg_main_t * pg, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number (), + vlib_get_thread_index (), si->sw_if_index, n_buffers, length_sum); } @@ -1083,6 +1084,7 @@ init_replay_buffers_inline (vlib_main_t * vm, vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; d0 = vec_elt (s->replay_packet_templates, i); + vnet_buffer2 (b0)->pg_replay_timestamp = s->replay_packet_timestamps[i]; n0 = n_data; if (data_offset + n_data >= vec_len (d0)) @@ -1090,7 +1092,7 @@ init_replay_buffers_inline (vlib_main_t * vm, b0->current_length = n0; - clib_memcpy (b0->data, d0 + data_offset, n0); + clib_memcpy_fast (b0->data, d0 + data_offset, n0); i = i + 1 == l ? 0 : i + 1; } } @@ -1147,8 +1149,8 @@ init_buffers_inline (vlib_main_t * vm, if (set_data) { - clib_memcpy (b0->data, data, n_data); - clib_memcpy (b1->data, data, n_data); + clib_memcpy_fast (b0->data, data, n_data); + clib_memcpy_fast (b1->data, data, n_data); } else { @@ -1172,31 +1174,12 @@ init_buffers_inline (vlib_main_t * vm, vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; if (set_data) - clib_memcpy (b0->data, data, n_data); + clib_memcpy_fast (b0->data, data, n_data); else ASSERT (validate_buffer_data2 (b0, s, data_offset, n_data)); } } -static void -pg_buffer_init (vlib_main_t * vm, - vlib_buffer_free_list_t * fl, u32 * buffers, u32 n_buffers) -{ - pg_main_t *pg = &pg_main; - pg_stream_t *s; - uword bi, si; - - si = fl->buffer_init_function_opaque & pow2_mask (24); - bi = fl->buffer_init_function_opaque >> 24; - - s = pool_elt_at_index (pg->streams, si); - - init_buffers_inline (vm, s, buffers, n_buffers, - /* data_offset */ bi * s->buffer_bytes, - /* n_data */ s->buffer_bytes, - /* set_data */ 1); -} - static u32 pg_stream_fill_helper (pg_main_t * pg, pg_stream_t * s, @@ -1204,30 +1187,10 @@ pg_stream_fill_helper (pg_main_t * pg, u32 * buffers, u32 * next_buffers, u32 n_alloc) { vlib_main_t *vm = vlib_get_main (); - vlib_buffer_free_list_t *f; uword is_start_of_packet = bi == s->buffer_indices; u32 n_allocated; - f = vlib_buffer_get_free_list (vm, bi->free_list_index); - - /* - * Historically, the pg maintained its own free lists and - * device drivers tx paths would return pkts. With the DPDK, - * that doesn't happen. - */ - if (DPDK == 0 && !(s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE)) - f->buffer_init_function = pg_buffer_init; - f->buffer_init_function_opaque = - (s - pg->streams) | ((bi - s->buffer_indices) << 24); - - if (is_start_of_packet) - vnet_buffer (&f->buffer_init_template)->sw_if_index[VLIB_RX] - = vnet_main.local_interface_sw_if_index; - - n_allocated = vlib_buffer_alloc_from_free_list (vm, - buffers, - n_alloc, - bi->free_list_index); + n_allocated = vlib_buffer_alloc (vm, buffers, n_alloc); if (n_allocated == 0) return 0; @@ -1238,15 +1201,12 @@ pg_stream_fill_helper (pg_main_t * pg, n_alloc = n_allocated; /* Reinitialize buffers */ - if (DPDK == 0 || CLIB_DEBUG > 0 - || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE)) - init_buffers_inline - (vm, s, - buffers, - n_alloc, (bi - s->buffer_indices) * s->buffer_bytes /* data offset */ , - s->buffer_bytes, - /* set_data */ - DPDK == 1 || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE) != 0); + init_buffers_inline + (vm, s, + buffers, + n_alloc, (bi - s->buffer_indices) * s->buffer_bytes /* data offset */ , + s->buffer_bytes, + /* set_data */ 1); if (next_buffers) pg_set_next_buffer_pointers (pg, s, buffers, next_buffers, n_alloc); @@ -1265,7 +1225,7 @@ pg_stream_fill_helper (pg_main_t * pg, l += vlib_buffer_index_length_in_chain (vm, buffers[i]); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number (), + vlib_get_thread_index (), si->sw_if_index, n_alloc, l); s->current_replay_packet_index += n_alloc; s->current_replay_packet_index %= @@ -1372,6 +1332,7 @@ typedef struct u32 stream_index; u32 packet_length; + u32 sw_if_index; /* Use pre data for packet data. */ vlib_buffer_t buffer; @@ -1386,7 +1347,7 @@ format_pg_input_trace (u8 * s, va_list * va) pg_main_t *pg = &pg_main; pg_stream_t *stream; vlib_node_t *n; - uword indent = format_get_indent (s); + u32 indent = format_get_indent (s); stream = 0; if (!pool_is_free_index (pg->streams, t->stream_index)) @@ -1398,9 +1359,10 @@ format_pg_input_trace (u8 * s, va_list * va) s = format (s, "stream %d", t->stream_index); s = format (s, ", %d bytes", t->packet_length); + s = format (s, ", %d sw_if_index", t->sw_if_index); s = format (s, "\n%U%U", - format_white_space, indent, format_vlib_buffer, &t->buffer); + format_white_space, indent, format_vnet_buffer, &t->buffer); s = format (s, "\n%U", format_white_space, indent); @@ -1457,13 +1419,18 @@ pg_input_trace (pg_main_t * pg, t0->packet_length = vlib_buffer_length_in_chain (vm, b0); t1->packet_length = vlib_buffer_length_in_chain (vm, b1); - clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data)); - clib_memcpy (&t1->buffer, b1, sizeof (b1[0]) - sizeof (b1->pre_data)); + t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX]; - clib_memcpy (t0->buffer.pre_data, b0->data, - sizeof (t0->buffer.pre_data)); - clib_memcpy (t1->buffer.pre_data, b1->data, - sizeof (t1->buffer.pre_data)); + clib_memcpy_fast (&t0->buffer, b0, + sizeof (b0[0]) - sizeof (b0->pre_data)); + clib_memcpy_fast (&t1->buffer, b1, + sizeof (b1[0]) - sizeof (b1->pre_data)); + + clib_memcpy_fast (t0->buffer.pre_data, b0->data, + sizeof (t0->buffer.pre_data)); + clib_memcpy_fast (t1->buffer.pre_data, b1->data, + sizeof (t1->buffer.pre_data)); } while (n_left >= 1) @@ -1483,9 +1450,11 @@ pg_input_trace (pg_main_t * pg, t0->stream_index = stream_index; t0->packet_length = vlib_buffer_length_in_chain (vm, b0); - clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data)); - clib_memcpy (t0->buffer.pre_data, b0->data, - sizeof (t0->buffer.pre_data)); + t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + clib_memcpy_fast (&t0->buffer, b0, + sizeof (b0[0]) - sizeof (b0->pre_data)); + clib_memcpy_fast (t0->buffer.pre_data, b0->data, + sizeof (t0->buffer.pre_data)); } } @@ -1525,7 +1494,24 @@ pg_generate_packets (vlib_node_runtime_t * node, { u32 *head, *start, *end; - vlib_get_next_frame (vm, node, next_index, to_next, n_left); + if (PREDICT_TRUE (next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT)) + { + vlib_next_frame_t *nf; + vlib_frame_t *f; + ethernet_input_frame_t *ef; + pg_interface_t *pi; + vlib_get_new_next_frame (vm, node, next_index, to_next, n_left); + nf = vlib_node_runtime_get_next_frame (vm, node, next_index); + f = vlib_get_frame (vm, nf->frame_index); + f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX; + + ef = vlib_frame_scalar_args (f); + pi = pool_elt_at_index (pg->interfaces, s->pg_if_index); + ef->sw_if_index = pi->sw_if_index; + ef->hw_if_index = pi->hw_if_index; + } + else + vlib_get_next_frame (vm, node, next_index, to_next, n_left); n_this_frame = n_packets_to_generate; if (n_this_frame > n_left) @@ -1536,12 +1522,13 @@ pg_generate_packets (vlib_node_runtime_t * node, head = clib_fifo_head (bi0->buffer_fifo); if (head + n_this_frame <= end) - vlib_copy_buffers (to_next, head, n_this_frame); + clib_memcpy_fast (to_next, head, n_this_frame * sizeof (u32)); else { u32 n = end - head; - vlib_copy_buffers (to_next + 0, head, n); - vlib_copy_buffers (to_next + n, start, n_this_frame - n); + clib_memcpy_fast (to_next + 0, head, n * sizeof (u32)); + clib_memcpy_fast (to_next + n, start, + (n_this_frame - n) * sizeof (u32)); } vec_foreach (bi, s->buffer_indices) @@ -1552,11 +1539,8 @@ pg_generate_packets (vlib_node_runtime_t * node, { vlib_buffer_t *b; b = vlib_get_buffer (vm, to_next[i]); - vnet_buffer (b)->device_input_feat.saved_next_index = - s->next_index; - vnet_buffer (b)->device_input_feat.buffer_advance = 0; b->current_config_index = current_config_index; - b->feature_arc_index = feature_arc_index; + vnet_buffer (b)->feature_arc_index = feature_arc_index; } n_trace = vlib_get_trace_count (vm, node);