2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * pg_stream.c: packet generator streams
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
40 #include <vnet/vnet.h>
41 #include <vnet/pg/pg.h>
42 #include <vnet/ethernet/ethernet.h>
43 #include <vnet/ip/ip.h>
44 #include <vnet/mpls/mpls.h>
45 #include <vnet/devices/devices.h>
47 /* Mark stream active or inactive. */
49 pg_stream_enable_disable (pg_main_t * pg, pg_stream_t * s, int want_enabled)
52 vnet_main_t *vnm = vnet_get_main ();
53 pg_interface_t *pi = pool_elt_at_index (pg->interfaces, s->pg_if_index);
55 want_enabled = want_enabled != 0;
57 if (pg_stream_is_enabled (s) == want_enabled)
58 /* No change necessary. */
62 s->n_packets_generated = 0;
64 /* Toggle enabled flag. */
65 s->flags ^= PG_STREAM_FLAGS_IS_ENABLED;
67 ASSERT (!pool_is_free (pg->streams, s));
69 vec_validate (pg->enabled_streams, s->worker_index);
70 pg->enabled_streams[s->worker_index] =
71 clib_bitmap_set (pg->enabled_streams[s->worker_index], s - pg->streams,
76 vnet_hw_interface_set_flags (vnm, pi->hw_if_index,
77 VNET_HW_INTERFACE_FLAG_LINK_UP);
79 vnet_sw_interface_set_flags (vnm, pi->sw_if_index,
80 VNET_SW_INTERFACE_FLAG_ADMIN_UP);
83 if (vlib_num_workers ())
84 vm = vlib_get_worker_vlib_main (s->worker_index);
86 vm = vlib_get_main ();
88 vlib_node_set_state (vm, pg_input_node.index,
90 (pg->enabled_streams[s->worker_index]) ?
91 VLIB_NODE_STATE_DISABLED : VLIB_NODE_STATE_POLLING));
93 s->packet_accumulator = 0;
94 s->time_last_generate = 0;
98 format_pg_output_trace (u8 * s, va_list * va)
100 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
101 CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
102 pg_output_trace_t *t = va_arg (*va, pg_output_trace_t *);
103 u32 indent = format_get_indent (s);
105 s = format (s, "%Ubuffer 0x%x: %U", format_white_space, indent,
106 t->buffer_index, format_vnet_buffer_no_chain, &t->buffer);
108 s = format (s, "\n%U%U", format_white_space, indent,
109 format_ethernet_header_with_length, t->buffer.pre_data,
110 sizeof (t->buffer.pre_data));
116 format_pg_interface_name (u8 * s, va_list * args)
118 pg_main_t *pg = &pg_main;
119 u32 if_index = va_arg (*args, u32);
122 pi = pool_elt_at_index (pg->interfaces, if_index);
123 s = format (s, "pg%d", pi->id);
128 static clib_error_t *
129 pg_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
133 if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
134 hw_flags = VNET_HW_INTERFACE_FLAG_LINK_UP;
136 vnet_hw_interface_set_flags (vnm, hw_if_index, hw_flags);
142 pg_mac_address_cmp (const mac_address_t * m1, const mac_address_t * m2)
144 return (!mac_address_cmp (m1, m2));
147 static clib_error_t *
148 pg_add_del_mac_address (vnet_hw_interface_t * hi,
149 const u8 * address, u8 is_add)
151 pg_main_t *pg = &pg_main;
153 if (ethernet_address_cast (address))
158 pi = pool_elt_at_index (pg->interfaces, hi->dev_instance);
160 mac_address_from_bytes (&mac, address);
162 vec_add1 (pi->allowed_mcast_macs, mac);
165 u32 pos = vec_search_with_function (pi->allowed_mcast_macs, &mac,
168 vec_del1 (pi->allowed_mcast_macs, pos);
175 VNET_DEVICE_CLASS (pg_dev_class) = {
177 .tx_function = pg_output,
178 .format_device_name = format_pg_interface_name,
179 .format_tx_trace = format_pg_output_trace,
180 .admin_up_down_function = pg_interface_admin_up_down,
181 .mac_addr_add_del_function = pg_add_del_mac_address,
186 pg_build_rewrite (vnet_main_t * vnm,
188 vnet_link_t link_type, const void *dst_address)
193 vec_validate (rewrite, sizeof (*h) - 1);
195 h[0] = clib_host_to_net_u16 (vnet_link_to_l3_proto (link_type));
201 VNET_HW_INTERFACE_CLASS (pg_interface_class,static) = {
202 .name = "Packet generator",
203 .build_rewrite = pg_build_rewrite,
208 pg_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags)
210 /* nothing for now */
215 pg_interface_enable_disable_coalesce (pg_interface_t * pi, u8 enable,
220 gro_flow_table_init (&pi->flow_table, 1 /* is_l2 */ ,
222 pi->coalesce_enabled = 1;
226 pi->coalesce_enabled = 0;
227 gro_flow_table_free (pi->flow_table);
232 format_pg_tun_tx_trace (u8 *s, va_list *args)
234 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
235 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
237 s = format (s, "PG: tunnel (no-encap)");
241 VNET_HW_INTERFACE_CLASS (pg_tun_hw_interface_class) = {
243 //.format_header = format_gre_header_with_length,
244 //.unformat_header = unformat_gre_header,
245 .build_rewrite = NULL,
246 //.update_adjacency = gre_update_adj,
247 .flags = VNET_HW_INTERFACE_CLASS_FLAG_P2P,
251 pg_interface_add_or_get (pg_main_t *pg, uword if_id, u8 gso_enabled,
252 u32 gso_size, u8 coalesce_enabled,
253 pg_interface_mode_t mode)
255 vnet_main_t *vnm = vnet_get_main ();
256 vlib_main_t *vm = vlib_get_main ();
258 vnet_hw_interface_t *hi;
262 p = hash_get (pg->if_index_by_if_id, if_id);
271 f64 now = vlib_time_now (vm);
274 pool_get (pg->interfaces, pi);
275 i = pi - pg->interfaces;
277 rnd = (u32) (now * 1e6);
278 rnd = random_u32 (&rnd);
279 clib_memcpy_fast (hw_addr + 2, &rnd, sizeof (rnd));
288 case PG_MODE_ETHERNET:
289 ethernet_register_interface (vnm, pg_dev_class.index, i, hw_addr,
290 &pi->hw_if_index, pg_eth_flag_change);
294 pi->hw_if_index = vnet_register_interface (
295 vnm, pg_dev_class.index, i, pg_tun_hw_interface_class.index, i);
298 hi = vnet_get_hw_interface (vnm, pi->hw_if_index);
301 hi->caps |= VNET_HW_INTERFACE_CAP_SUPPORTS_TCP_GSO;
303 pi->gso_size = gso_size;
304 if (coalesce_enabled)
306 pg_interface_enable_disable_coalesce (pi, 1, hi->tx_node_index);
309 pi->sw_if_index = hi->sw_if_index;
311 hash_set (pg->if_index_by_if_id, if_id, i);
313 vec_validate (pg->if_id_by_sw_if_index, hi->sw_if_index);
314 pg->if_id_by_sw_if_index[hi->sw_if_index] = i;
316 if (vlib_num_workers ())
318 pi->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
319 CLIB_CACHE_LINE_BYTES);
328 do_edit (pg_stream_t * stream,
329 pg_edit_group_t * g, pg_edit_t * e, uword want_commit)
331 u32 i, i0, i1, mask, n_bits_left;
334 i0 = e->lsb_bit_offset / BITS (u8);
336 /* Make space for edit in value and mask. */
337 vec_validate (g->fixed_packet_data, i0);
338 vec_validate (g->fixed_packet_data_mask, i0);
340 if (e->type != PG_EDIT_FIXED)
345 case PG_EDIT_INCREMENT:
346 e->last_increment_value = pg_edit_get_value (e, PG_EDIT_LO);
355 ASSERT (e->type != PG_EDIT_INVALID_TYPE);
356 vec_add1 (g->non_fixed_edits, e[0]);
361 s = g->fixed_packet_data;
362 m = g->fixed_packet_data_mask;
364 n_bits_left = e->n_bits;
365 i0 = e->lsb_bit_offset / BITS (u8);
366 i1 = e->lsb_bit_offset % BITS (u8);
368 v = e->values[PG_EDIT_LO];
369 i = pg_edit_n_alloc_bytes (e) - 1;
371 /* Odd low order bits?. */
372 if (i1 != 0 && n_bits_left > 0)
374 u32 n = clib_min (n_bits_left, BITS (u8) - i1);
376 mask = pow2_mask (n) << i1;
378 ASSERT (i0 < vec_len (s));
379 ASSERT (i < vec_len (v));
380 ASSERT ((v[i] & ~mask) == 0);
382 s[i0] |= v[i] & mask;
391 while (n_bits_left >= 8)
393 ASSERT (i0 < vec_len (s));
394 ASSERT (i < vec_len (v));
404 /* Odd high order bits. */
407 mask = pow2_mask (n_bits_left);
409 ASSERT (i0 < vec_len (s));
410 ASSERT (i < vec_len (v));
411 ASSERT ((v[i] & ~mask) == 0);
413 s[i0] |= v[i] & mask;
422 pg_edit_group_get_fixed_packet_data (pg_stream_t * s,
425 void *packet_data_mask)
427 pg_edit_group_t *g = pg_stream_get_group (s, group_index);
430 vec_foreach (e, g->edits) do_edit (s, g, e, /* want_commit */ 0);
432 clib_memcpy_fast (packet_data, g->fixed_packet_data,
433 vec_len (g->fixed_packet_data));
434 clib_memcpy_fast (packet_data_mask, g->fixed_packet_data_mask,
435 vec_len (g->fixed_packet_data_mask));
439 perform_fixed_edits (pg_stream_t * s)
445 for (i = vec_len (s->edit_groups) - 1; i >= 0; i--)
447 g = vec_elt_at_index (s->edit_groups, i);
448 vec_foreach (e, g->edits) do_edit (s, g, e, /* want_commit */ 1);
450 /* All edits have either been performed or added to
451 g->non_fixed_edits. So, we can delete the vector. */
455 vec_free (s->fixed_packet_data_mask);
456 vec_free (s->fixed_packet_data);
457 vec_foreach (g, s->edit_groups)
460 g->start_byte_offset = vec_len (s->fixed_packet_data);
462 /* Relocate and copy non-fixed edits from group to stream. */
463 vec_foreach (e, g->non_fixed_edits)
464 e->lsb_bit_offset += g->start_byte_offset * BITS (u8);
466 for (i = 0; i < vec_len (g->non_fixed_edits); i++)
467 ASSERT (g->non_fixed_edits[i].type != PG_EDIT_INVALID_TYPE);
469 vec_add (s->non_fixed_edits,
470 g->non_fixed_edits, vec_len (g->non_fixed_edits));
471 vec_free (g->non_fixed_edits);
473 vec_add (s->fixed_packet_data,
474 g->fixed_packet_data, vec_len (g->fixed_packet_data));
475 vec_add (s->fixed_packet_data_mask,
476 g->fixed_packet_data_mask, vec_len (g->fixed_packet_data_mask));
481 pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
483 vlib_main_t *vm = vlib_get_main ();
487 if (!pg->stream_index_by_name)
488 pg->stream_index_by_name
489 = hash_create_vec (0, sizeof (s->name[0]), sizeof (uword));
491 /* Delete any old stream with the same name. */
493 && (p = hash_get_mem (pg->stream_index_by_name, s_init->name)))
495 pg_stream_del (pg, p[0]);
498 pool_get (pg->streams, s);
501 /* Give it a name. */
503 s->name = format (0, "stream%d", s - pg->streams);
505 s->name = vec_dup (s->name);
507 hash_set_mem (pg->stream_index_by_name, s->name, s - pg->streams);
509 /* Get fixed part of buffer data. */
511 perform_fixed_edits (s);
513 /* Determine packet size. */
514 switch (s->packet_size_edit_type)
516 case PG_EDIT_INCREMENT:
518 if (s->min_packet_bytes == s->max_packet_bytes)
519 s->packet_size_edit_type = PG_EDIT_FIXED;
523 /* Get packet size from fixed edits. */
524 s->packet_size_edit_type = PG_EDIT_FIXED;
525 if (!s->replay_packet_templates)
526 s->min_packet_bytes = s->max_packet_bytes =
527 vec_len (s->fixed_packet_data);
531 s->last_increment_packet_size = s->min_packet_bytes;
536 s->buffer_bytes = vlib_buffer_get_default_data_size (vm);
537 n = s->max_packet_bytes / s->buffer_bytes;
538 n += (s->max_packet_bytes % s->buffer_bytes) != 0;
540 vec_resize (s->buffer_indices, n);
543 /* Find an interface to use. */
544 s->pg_if_index = pg_interface_add_or_get (
545 pg, s->if_id, 0 /* gso_enabled */, 0 /* gso_size */,
546 0 /* coalesce_enabled */, PG_MODE_ETHERNET);
548 if (s->sw_if_index[VLIB_RX] == ~0)
550 pg_interface_t *pi = pool_elt_at_index (pg->interfaces, s->pg_if_index);
552 * Default the RX interface if unset. It's a bad mistake to
553 * set [VLIB_TX] prior to ip lookup, since the ip lookup code
554 * interprets [VLIB_TX] as a fib index...
556 s->sw_if_index[VLIB_RX] = pi->sw_if_index;
559 /* Connect the graph. */
560 s->next_index = vlib_node_add_next (vm, device_input_node.index,
565 pg_stream_del (pg_main_t * pg, uword index)
568 pg_buffer_index_t *bi;
570 s = pool_elt_at_index (pg->streams, index);
572 pg_stream_enable_disable (pg, s, /* want_enabled */ 0);
573 hash_unset_mem (pg->stream_index_by_name, s->name);
575 vec_foreach (bi, s->buffer_indices)
577 clib_fifo_free (bi->buffer_fifo);
581 pool_put (pg->streams, s);
585 pg_stream_change (pg_main_t * pg, pg_stream_t * s)
587 /* Determine packet size. */
588 switch (s->packet_size_edit_type)
590 case PG_EDIT_INCREMENT:
592 if (s->min_packet_bytes == s->max_packet_bytes)
593 s->packet_size_edit_type = PG_EDIT_FIXED;
598 /* Get packet size from fixed edits. */
599 s->packet_size_edit_type = PG_EDIT_FIXED;
600 if (!s->replay_packet_templates)
601 s->min_packet_bytes = s->max_packet_bytes =
602 vec_len (s->fixed_packet_data);
606 s->last_increment_packet_size = s->min_packet_bytes;
611 * fd.io coding-style-patch-verification: ON
614 * eval: (c-set-style "gnu")