#include <vlib/vlib.h>
#include <vnet/pg/pg.h>
#include <vnet/vnet.h>
+#include <vnet/ethernet/ethernet.h>
#include <vnet/feature/feature.h>
#include <vnet/devices/devices.h>
vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
d0 = vec_elt (s->replay_packet_templates, i);
+ vnet_buffer2 (b0)->pg_replay_timestamp = s->replay_packet_timestamps[i];
n0 = n_data;
if (data_offset + n_data >= vec_len (d0))
b0->current_length = n0;
- clib_memcpy (b0->data, d0 + data_offset, n0);
+ clib_memcpy_fast (b0->data, d0 + data_offset, n0);
i = i + 1 == l ? 0 : i + 1;
}
}
if (set_data)
{
- clib_memcpy (b0->data, data, n_data);
- clib_memcpy (b1->data, data, n_data);
+ clib_memcpy_fast (b0->data, data, n_data);
+ clib_memcpy_fast (b1->data, data, n_data);
}
else
{
vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
if (set_data)
- clib_memcpy (b0->data, data, n_data);
+ clib_memcpy_fast (b0->data, data, n_data);
else
ASSERT (validate_buffer_data2 (b0, s, data_offset, n_data));
}
}
-static void
-pg_buffer_init (vlib_main_t * vm,
- vlib_buffer_free_list_t * fl, u32 * buffers, u32 n_buffers)
-{
- pg_main_t *pg = &pg_main;
- pg_stream_t *s;
- uword bi, si;
-
- si = fl->buffer_init_function_opaque & pow2_mask (24);
- bi = fl->buffer_init_function_opaque >> 24;
-
- s = pool_elt_at_index (pg->streams, si);
-
- init_buffers_inline (vm, s, buffers, n_buffers,
- /* data_offset */ bi * s->buffer_bytes,
- /* n_data */ s->buffer_bytes,
- /* set_data */ 1);
-}
-
static u32
pg_stream_fill_helper (pg_main_t * pg,
pg_stream_t * s,
u32 * buffers, u32 * next_buffers, u32 n_alloc)
{
vlib_main_t *vm = vlib_get_main ();
- vlib_buffer_free_list_t *f;
uword is_start_of_packet = bi == s->buffer_indices;
u32 n_allocated;
- f = vlib_buffer_get_free_list (vm, bi->free_list_index);
-
- /*
- * Historically, the pg maintained its own free lists and
- * device drivers tx paths would return pkts.
- */
- if (vm->buffer_main->callbacks_registered == 0 &&
- !(s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE))
- f->buffer_init_function = pg_buffer_init;
- f->buffer_init_function_opaque =
- (s - pg->streams) | ((bi - s->buffer_indices) << 24);
-
- if (is_start_of_packet)
- vnet_buffer (&f->buffer_init_template)->sw_if_index[VLIB_RX]
- = vnet_main.local_interface_sw_if_index;
-
- n_allocated = vlib_buffer_alloc_from_free_list (vm,
- buffers,
- n_alloc,
- bi->free_list_index);
+ n_allocated = vlib_buffer_alloc (vm, buffers, n_alloc);
if (n_allocated == 0)
return 0;
n_alloc = n_allocated;
/* Reinitialize buffers */
- if (vm->buffer_main->callbacks_registered == 0 || CLIB_DEBUG > 0
- || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE))
- init_buffers_inline
- (vm, s,
- buffers,
- n_alloc, (bi - s->buffer_indices) * s->buffer_bytes /* data offset */ ,
- s->buffer_bytes,
- /* set_data */
- vm->buffer_main->callbacks_registered != 0
- || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE) != 0);
+ init_buffers_inline
+ (vm, s,
+ buffers,
+ n_alloc, (bi - s->buffer_indices) * s->buffer_bytes /* data offset */ ,
+ s->buffer_bytes,
+ /* set_data */ 1);
if (next_buffers)
pg_set_next_buffer_pointers (pg, s, buffers, next_buffers, n_alloc);
t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX];
- clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
- clib_memcpy (&t1->buffer, b1, sizeof (b1[0]) - sizeof (b1->pre_data));
+ clib_memcpy_fast (&t0->buffer, b0,
+ sizeof (b0[0]) - sizeof (b0->pre_data));
+ clib_memcpy_fast (&t1->buffer, b1,
+ sizeof (b1[0]) - sizeof (b1->pre_data));
- clib_memcpy (t0->buffer.pre_data, b0->data,
- sizeof (t0->buffer.pre_data));
- clib_memcpy (t1->buffer.pre_data, b1->data,
- sizeof (t1->buffer.pre_data));
+ clib_memcpy_fast (t0->buffer.pre_data, b0->data,
+ sizeof (t0->buffer.pre_data));
+ clib_memcpy_fast (t1->buffer.pre_data, b1->data,
+ sizeof (t1->buffer.pre_data));
}
while (n_left >= 1)
t0->stream_index = stream_index;
t0->packet_length = vlib_buffer_length_in_chain (vm, b0);
t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
- clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
- clib_memcpy (t0->buffer.pre_data, b0->data,
- sizeof (t0->buffer.pre_data));
+ clib_memcpy_fast (&t0->buffer, b0,
+ sizeof (b0[0]) - sizeof (b0->pre_data));
+ clib_memcpy_fast (t0->buffer.pre_data, b0->data,
+ sizeof (t0->buffer.pre_data));
}
}
{
u32 *head, *start, *end;
- vlib_get_next_frame (vm, node, next_index, to_next, n_left);
+ if (PREDICT_TRUE (next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT))
+ {
+ vlib_next_frame_t *nf;
+ vlib_frame_t *f;
+ ethernet_input_frame_t *ef;
+ pg_interface_t *pi;
+ vlib_get_new_next_frame (vm, node, next_index, to_next, n_left);
+ nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
+ f = vlib_get_frame (vm, nf->frame_index);
+ f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX;
+
+ ef = vlib_frame_scalar_args (f);
+ pi = pool_elt_at_index (pg->interfaces, s->pg_if_index);
+ ef->sw_if_index = pi->sw_if_index;
+ ef->hw_if_index = pi->hw_if_index;
+ }
+ else
+ vlib_get_next_frame (vm, node, next_index, to_next, n_left);
n_this_frame = n_packets_to_generate;
if (n_this_frame > n_left)
head = clib_fifo_head (bi0->buffer_fifo);
if (head + n_this_frame <= end)
- vlib_copy_buffers (to_next, head, n_this_frame);
+ clib_memcpy_fast (to_next, head, n_this_frame * sizeof (u32));
else
{
u32 n = end - head;
- vlib_copy_buffers (to_next + 0, head, n);
- vlib_copy_buffers (to_next + n, start, n_this_frame - n);
+ clib_memcpy_fast (to_next + 0, head, n * sizeof (u32));
+ clib_memcpy_fast (to_next + n, start,
+ (n_this_frame - n) * sizeof (u32));
}
vec_foreach (bi, s->buffer_indices)
{
vlib_buffer_t *b;
b = vlib_get_buffer (vm, to_next[i]);
- vnet_buffer (b)->device_input_feat.saved_next_index =
- s->next_index;
- vnet_buffer (b)->device_input_feat.buffer_advance = 0;
b->current_config_index = current_config_index;
- b->feature_arc_index = feature_arc_index;
+ vnet_buffer (b)->feature_arc_index = feature_arc_index;
}
n_trace = vlib_get_trace_count (vm, node);
Code Review / vpp.git / blobdiff