} \
__VA_ARGS__ vlib_thread_registration_t x
+always_inline u32
+vlib_num_workers ()
+{
+ return vlib_thread_main.n_vlib_mains - 1;
+}
+
+always_inline u32
+vlib_get_worker_cpu_index (u32 worker_index)
+{
+ return worker_index + 1;
+}
+
+always_inline vlib_main_t *
+vlib_get_worker_vlib_main (u32 worker_index)
+{
+ vlib_main_t *vm;
+ vlib_thread_main_t *tm = &vlib_thread_main;
+ ASSERT (worker_index < tm->n_vlib_mains - 1);
+ vm = vlib_mains[worker_index + 1];
+ ASSERT (vm);
+ return vm;
+}
+
#endif /* included_vlib_threads_h */
/*
u32 * packets,
u32 n_packets)
{
- vlib_main_t * vm = pg->vlib_main;
+ vlib_main_t * vm = vlib_get_main();
u32 ip_offset, icmp_offset;
icmp_offset = g->start_byte_offset;
u32 * packets,
u32 n_packets)
{
- vlib_main_t * vm = pg->vlib_main;
+ vlib_main_t * vm = vlib_get_main();
u32 ip_offset, icmp_offset;
int bogus_length;
u32 * packets,
u32 n_packets)
{
- vlib_main_t * vm = pg->vlib_main;
+ vlib_main_t * vm = vlib_get_main();
u32 ip_offset;
ip_offset = g->start_byte_offset;
u32 * packets,
u32 n_packets)
{
- vlib_main_t * vm = pg->vlib_main;
+ vlib_main_t * vm = vlib_get_main();
u32 ip_header_offset = g->start_byte_offset;
while (n_packets >= 2)
u32 n_packets,
u32 flags)
{
- vlib_main_t * vm = pg->vlib_main;
+ vlib_main_t * vm = vlib_get_main();
u32 ip_offset, udp_offset;
udp_offset = g->start_byte_offset;
v = format (v, "buffer-size %d, ", t->buffer_bytes);
+ v = format (v, "worker %d, ", t->worker_index);
+
if (v)
{
s = format (s, " %v", v);
unformat_vlib_node, vm, &s.node_index))
;
+ else if (unformat (input, "worker %u", &s.worker_index))
+ ;
+
else if (unformat (input, "interface %U",
unformat_vnet_sw_interface, vnm,
&s.sw_if_index[VLIB_RX]))
else
n = 0;
+ if (s.worker_index >= vlib_num_workers ())
+ s.worker_index = 0;
+
if (pcap_file_name != 0)
{
error = pg_pcap_read (&s, pcap_file_name);
unformat_pg_payload (unformat_input_t * input, va_list * args)
{
pg_stream_t *s = va_arg (*args, pg_stream_t *);
- pg_main_t *pg = &pg_main;
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
pg_edit_t *e;
u32 i, node_index, len, max_len;
u8 *v;
clib_error_t *error;
pg_main_t *pg = &pg_main;
- pg->vlib_main = vm;
pg->if_index_by_if_id = hash_create (0, sizeof (uword));
if ((error = vlib_call_init_function (vm, vnet_main_init)))
u32 n_bits,
u32 byte_offset, u32 is_net_byte_order, u64 v_min, u64 v_max)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
while (n_buffers >= 4)
{
u32 is_net_byte_order,
u32 want_sum, u64 * sum_result, u64 v_min, u64 v_max, u64 v)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
u64 sum = 0;
ASSERT (v >= v_min && v <= v_max);
u32 is_net_byte_order,
u32 want_sum, u64 * sum_result, u64 v_min, u64 v_max)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
u64 v_diff = v_max - v_min + 1;
u64 r_mask = max_pow2 (v_diff) - 1;
u64 v0, v1;
u32 n_bits,
u32 byte_offset, u64 v_min, u64 v_max, u64 mask, u32 shift)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
while (n_buffers >= 4)
{
u32 byte_offset,
u64 v_min, u64 v_max, u64 v, u64 mask, u32 shift)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
ASSERT (v >= v_min && v <= v_max);
u32 n_bits,
u32 byte_offset, u64 v_min, u64 v_max, u64 mask, u32 shift)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
u64 v_diff = v_max - v_min + 1;
u64 r_mask = max_pow2 (v_diff) - 1;
u64 v0, v1;
pg_stream_t * s,
u32 * buffers, u32 n_buffers)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
pg_buffer_index_t *pbi;
uword n_bytes_left;
static u32 *unused_buffers = 0;
pg_stream_t * s,
u32 * buffers, u32 * next_buffers, u32 n_buffers)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
while (n_buffers >= 4)
{
pg_buffer_index_t * bi,
u32 * buffers, u32 * next_buffers, u32 n_alloc)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
vlib_buffer_free_list_t *f;
uword is_start_of_packet = bi == s->buffer_indices;
u32 n_allocated;
vlib_node_runtime_t * node,
pg_stream_t * s, u32 * buffers, u32 n_buffers)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
u32 *b, n_left, stream_index, next_index;
n_left = n_buffers;
pg_main_t * pg,
pg_stream_t * s, uword n_packets_to_generate)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
u32 *to_next, n_this_frame, n_left, n_trace, n_packets_in_fifo;
uword n_packets_generated;
pg_buffer_index_t *bi, *bi0;
static uword
pg_input_stream (vlib_node_runtime_t * node, pg_main_t * pg, pg_stream_t * s)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
uword n_packets;
f64 time_now, dt;
uword i;
pg_main_t *pg = &pg_main;
uword n_packets = 0;
+ u32 num_workers = vlib_num_workers ();
+ u32 cpu_index = os_get_cpu_number ();
/* *INDENT-OFF* */
clib_bitmap_foreach (i, pg->enabled_streams, ({
- n_packets += pg_input_stream (node, pg, vec_elt_at_index (pg->streams, i));
+ pg_stream_t *s = vec_elt_at_index (pg->streams, i);
+ if (num_workers == 0 ||
+ vlib_get_worker_cpu_index (s->worker_index) == cpu_index)
+ n_packets += pg_input_stream (node, pg, s);
}));
/* *INDENT-ON* */
vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
pg_interface_t *pif = pool_elt_at_index (pg->interfaces, rd->dev_instance);
+ if (PREDICT_FALSE (pif->lockp != 0))
+ while (__sync_lock_test_and_set (pif->lockp, 1))
+ ;
+
if (pif->pcap_file_name != 0)
{
while (n_left > 0)
}
vlib_buffer_free_no_next (vm, vlib_frame_args (frame), n_buffers);
+ if (PREDICT_FALSE (pif->lockp != 0))
+ *pif->lockp = 0;
return n_buffers;
}
/* Node where stream's buffers get put. */
u32 node_index;
+ /* Worker thread index */
+ u32 worker_index;
+
/* Output next index to reach output node from stream input node. */
u32 next_index;
typedef struct
{
+ /* TX lock */
+ volatile u32 *lockp;
+
/* VLIB interface indices. */
u32 hw_if_index, sw_if_index;
typedef struct pg_main_t
{
- /* Back pointer to main structure. */
- vlib_main_t *vlib_main;
-
/* Pool of streams. */
pg_stream_t *streams;
void
pg_stream_enable_disable (pg_main_t * pg, pg_stream_t * s, int want_enabled)
{
+ vlib_main_t *vm;
vnet_main_t *vnm = vnet_get_main ();
pg_interface_t *pi = pool_elt_at_index (pg->interfaces, s->pg_if_index);
VNET_SW_INTERFACE_FLAG_ADMIN_UP);
}
- vlib_node_set_state (pg->vlib_main,
- pg_input_node.index,
- (clib_bitmap_is_zero (pg->enabled_streams)
- ? VLIB_NODE_STATE_DISABLED
- : VLIB_NODE_STATE_POLLING));
+ if (vlib_num_workers ())
+ vm = vlib_get_worker_vlib_main (s->worker_index);
+ else
+ vm = vlib_get_main ();
+
+ vlib_node_set_state (vm, pg_input_node.index,
+ (clib_bitmap_is_zero (pg->enabled_streams) ?
+ VLIB_NODE_STATE_DISABLED : VLIB_NODE_STATE_POLLING));
s->packet_accumulator = 0;
s->time_last_generate = 0;
hash_set (pg->if_index_by_if_id, if_id, i);
+ if (vlib_num_workers ())
+ {
+ pi->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
+ CLIB_CACHE_LINE_BYTES);
+ *pi->lockp = 0;
+ }
+
ip4_sw_interface_enable_disable (pi->hw_if_index, 1);
ip6_sw_interface_enable_disable (pi->hw_if_index, 1);
mpls_sw_interface_enable_disable (&mpls_main, pi->hw_if_index, 1);
void
pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
pg_stream_t *s;
uword *p;
vec_resize (s->buffer_indices, n);
vec_foreach (bi, s->buffer_indices)
- bi->free_list_index = vlib_buffer_create_free_list (vm, s->buffer_bytes,
- "pg stream %d buffer #%d",
- s - pg->streams,
- 1 + (bi -
- s->
- buffer_indices));
+ {
+ vlib_main_t *vmt;
+ vmt =
+ vlib_num_workers ()? vlib_get_worker_vlib_main (s->worker_index) : vm;
+ bi->free_list_index =
+ vlib_buffer_create_free_list (vmt, s->buffer_bytes,
+ "pg stream %d buffer #%d",
+ s - pg->streams,
+ 1 + (bi - s->buffer_indices));
+ }
}
/* Find an interface to use. */
void
pg_stream_del (pg_main_t * pg, uword index)
{
- vlib_main_t *vm = pg->vlib_main;
+ vlib_main_t *vm = vlib_get_main ();
pg_stream_t *s;
pg_buffer_index_t *bi;
vec_foreach (bi, s->buffer_indices)
{
- vlib_buffer_delete_free_list (vm, bi->free_list_index);
+ vlib_buffer_delete_free_list (vlib_num_workers ()?
+ vlib_get_worker_vlib_main (s->worker_index)
+ : vm, bi->free_list_index);
clib_fifo_free (bi->buffer_fifo);
}