#include <vnet/devices/pipe/pipe.h>
#include <vppinfra/sparse_vec.h>
#include <vnet/l2/l2_bvi.h>
+#include <vnet/classify/trace_classify.h>
#define foreach_ethernet_input_next \
_ (PUNT, "error-punt") \
l->n_bytes += vlib_buffer_length_in_chain (vm, b);
}
+#define DMAC_MASK clib_net_to_host_u64 (0xFFFFFFFFFFFF0000)
+#define DMAC_IGBIT clib_net_to_host_u64 (0x0100000000000000)
+
+#ifdef CLIB_HAVE_VEC256
+static_always_inline u32
+is_dmac_bad_x4 (u64 * dmacs, u64 hwaddr)
+{
+ u64x4 r0 = u64x4_load_unaligned (dmacs) & u64x4_splat (DMAC_MASK);
+ r0 = (r0 != u64x4_splat (hwaddr)) & ((r0 & u64x4_splat (DMAC_IGBIT)) == 0);
+ return u8x32_msb_mask ((u8x32) (r0));
+}
+#else
+static_always_inline u8
+is_dmac_bad (u64 dmac, u64 hwaddr)
+{
+ u64 r0 = dmac & DMAC_MASK;
+ return (r0 != hwaddr) && ((r0 & DMAC_IGBIT) == 0);
+}
+#endif
+
+static_always_inline u8
+is_sec_dmac_bad (u64 dmac, u64 hwaddr)
+{
+ return ((dmac & DMAC_MASK) != hwaddr);
+}
+
+#ifdef CLIB_HAVE_VEC256
+static_always_inline u32
+is_sec_dmac_bad_x4 (u64 * dmacs, u64 hwaddr)
+{
+ u64x4 r0 = u64x4_load_unaligned (dmacs) & u64x4_splat (DMAC_MASK);
+ r0 = (r0 != u64x4_splat (hwaddr));
+ return u8x32_msb_mask ((u8x32) (r0));
+}
+#endif
+
+static_always_inline u8
+eth_input_sec_dmac_check_x1 (u64 hwaddr, u64 * dmac, u8 * dmac_bad)
+{
+ dmac_bad[0] &= is_sec_dmac_bad (dmac[0], hwaddr);
+ return dmac_bad[0];
+}
+
+static_always_inline u32
+eth_input_sec_dmac_check_x4 (u64 hwaddr, u64 * dmac, u8 * dmac_bad)
+{
+#ifdef CLIB_HAVE_VEC256
+ *(u32 *) (dmac_bad + 0) &= is_sec_dmac_bad_x4 (dmac + 0, hwaddr);
+#else
+ dmac_bad[0] &= is_sec_dmac_bad (dmac[0], hwaddr);
+ dmac_bad[1] &= is_sec_dmac_bad (dmac[1], hwaddr);
+ dmac_bad[2] &= is_sec_dmac_bad (dmac[2], hwaddr);
+ dmac_bad[3] &= is_sec_dmac_bad (dmac[3], hwaddr);
+#endif
+ return *(u32 *) dmac_bad;
+}
+
+static_always_inline void
+eth_input_process_frame_dmac_check (vnet_hw_interface_t * hi,
+ u64 * dmacs, u8 * dmacs_bad,
+ u32 n_packets, ethernet_interface_t * ei,
+ u8 have_sec_dmac)
+{
+ u64 hwaddr = (*(u64 *) hi->hw_address) & DMAC_MASK;
+ u64 *dmac = dmacs;
+ u8 *dmac_bad = dmacs_bad;
+ u32 bad = 0;
+ i32 n_left = n_packets;
+
+#ifdef CLIB_HAVE_VEC256
+ while (n_left > 0)
+ {
+ bad |= *(u32 *) (dmac_bad + 0) = is_dmac_bad_x4 (dmac + 0, hwaddr);
+ bad |= *(u32 *) (dmac_bad + 4) = is_dmac_bad_x4 (dmac + 4, hwaddr);
+
+ /* next */
+ dmac += 8;
+ dmac_bad += 8;
+ n_left -= 8;
+ }
+#else
+ while (n_left > 0)
+ {
+ bad |= dmac_bad[0] = is_dmac_bad (dmac[0], hwaddr);
+ bad |= dmac_bad[1] = is_dmac_bad (dmac[1], hwaddr);
+ bad |= dmac_bad[2] = is_dmac_bad (dmac[2], hwaddr);
+ bad |= dmac_bad[3] = is_dmac_bad (dmac[3], hwaddr);
+
+ /* next */
+ dmac += 4;
+ dmac_bad += 4;
+ n_left -= 4;
+ }
+#endif
+
+ if (have_sec_dmac && bad)
+ {
+ mac_address_t *addr;
+
+ vec_foreach (addr, ei->secondary_addrs)
+ {
+ u64 hwaddr = ((u64 *) addr)[0] & DMAC_MASK;
+ i32 n_left = n_packets;
+ u64 *dmac = dmacs;
+ u8 *dmac_bad = dmacs_bad;
+
+ bad = 0;
+
+ while (n_left > 0)
+ {
+ int adv = 0;
+ int n_bad;
+
+ /* skip any that have already matched */
+ if (!dmac_bad[0])
+ {
+ dmac += 1;
+ dmac_bad += 1;
+ n_left -= 1;
+ continue;
+ }
+
+ n_bad = clib_min (4, n_left);
+
+ /* If >= 4 left, compare 4 together */
+ if (n_bad == 4)
+ {
+ bad |= eth_input_sec_dmac_check_x4 (hwaddr, dmac, dmac_bad);
+ adv = 4;
+ n_bad = 0;
+ }
+
+ /* handle individually */
+ while (n_bad > 0)
+ {
+ bad |= eth_input_sec_dmac_check_x1 (hwaddr, dmac + adv,
+ dmac_bad + adv);
+ adv += 1;
+ n_bad -= 1;
+ }
+
+ dmac += adv;
+ dmac_bad += adv;
+ n_left -= adv;
+ }
+
+ if (!bad) /* can stop looping if everything matched */
+ break;
+ }
+ }
+}
+
/* process frame of buffers, store ethertype into array and update
buffer metadata fields depending on interface being l2 or l3 assuming that
packets are untagged. For tagged packets those fields are updated later.
i32 n_left = n_packets;
vlib_buffer_t *b[20];
u32 *from;
+ ethernet_interface_t *ei = ethernet_get_interface (em, hi->hw_if_index);
from = buffer_indices;
if (dmac_check)
{
- u64 mask = clib_net_to_host_u64 (0xFFFFFFFFFFFF0000);
- u64 igbit = clib_net_to_host_u64 (0x0100000000000000);
- u64 hwaddr = (*(u64 *) hi->hw_address) & mask;
- u64 *dmac = dmacs;
- u8 *dmac_bad = dmacs_bad;
-
- n_left = n_packets;
-
-#ifdef CLIB_HAVE_VEC256
- u64x4 igbit4 = u64x4_splat (igbit);
- u64x4 mask4 = u64x4_splat (mask);
- u64x4 hwaddr4 = u64x4_splat (hwaddr);
- while (n_left >= 0)
- {
- u64x4 r0, r1;
- r0 = u64x4_load_unaligned (dmac + 0) & mask4;
- r1 = u64x4_load_unaligned (dmac + 4) & mask4;
-
- r0 = (r0 != hwaddr4) & ((r0 & igbit4) == 0);
- r1 = (r1 != hwaddr4) & ((r1 & igbit4) == 0);
-
- *(u32 *) (dmac_bad + 0) = u8x32_msb_mask ((u8x32) (r0));
- *(u32 *) (dmac_bad + 4) = u8x32_msb_mask ((u8x32) (r1));
-
- /* next */
- dmac += 8;
- dmac_bad += 8;
- n_left -= 8;
- }
-#else
- while (n_left > 0)
- {
- u64 r0, r1, r2, r3;
- r0 = dmac[0] & mask;
- r1 = dmac[1] & mask;
- r2 = dmac[2] & mask;
- r3 = dmac[3] & mask;
-
- r0 = (r0 != hwaddr) && ((r0 & igbit) == 0);
- r1 = (r1 != hwaddr) && ((r1 & igbit) == 0);
- r2 = (r2 != hwaddr) && ((r2 & igbit) == 0);
- r3 = (r3 != hwaddr) && ((r3 & igbit) == 0);
-
- dmac_bad[0] = r0;
- dmac_bad[1] = r1;
- dmac_bad[2] = r2;
- dmac_bad[3] = r3;
-
- /* next */
- dmac += 4;
- dmac_bad += 4;
- n_left -= 4;
- }
-#endif
+ if (ei && vec_len (ei->secondary_addrs))
+ eth_input_process_frame_dmac_check (hi, dmacs, dmacs_bad, n_packets,
+ ei, 1 /* have_sec_dmac */ );
+ else
+ eth_input_process_frame_dmac_check (hi, dmacs, dmacs_bad, n_packets,
+ ei, 0 /* have_sec_dmac */ );
}
next_ip4 = em->l3_next.input_next_ip4;
vlib_frame_t * from_frame)
{
u32 *from, n_left;
- if ((node->flags & VLIB_NODE_FLAG_TRACE) == 0)
- return;
+ if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+ {
+ from = vlib_frame_vector_args (from_frame);
+ n_left = from_frame->n_vectors;
+
+ while (n_left)
+ {
+ ethernet_input_trace_t *t0;
+ vlib_buffer_t *b0 = vlib_get_buffer (vm, from[0]);
- from = vlib_frame_vector_args (from_frame);
- n_left = from_frame->n_vectors;
+ if (b0->flags & VLIB_BUFFER_IS_TRACED)
+ {
+ t0 = vlib_add_trace (vm, node, b0,
+ sizeof (ethernet_input_trace_t));
+ clib_memcpy_fast (t0->packet_data, b0->data + b0->current_data,
+ sizeof (t0->packet_data));
+ t0->frame_flags = from_frame->flags;
+ clib_memcpy_fast (&t0->frame_data,
+ vlib_frame_scalar_args (from_frame),
+ sizeof (ethernet_input_frame_t));
+ }
+ from += 1;
+ n_left -= 1;
+ }
+ }
- while (n_left)
+ /* rx pcap capture if enabled */
+ if (PREDICT_FALSE (vlib_global_main.pcap.pcap_rx_enable))
{
- ethernet_input_trace_t *t0;
- vlib_buffer_t *b0 = vlib_get_buffer (vm, from[0]);
+ u32 bi0;
+ vnet_pcap_t *pp = &vlib_global_main.pcap;
- if (b0->flags & VLIB_BUFFER_IS_TRACED)
+ from = vlib_frame_vector_args (from_frame);
+ n_left = from_frame->n_vectors;
+ while (n_left > 0)
{
- t0 = vlib_add_trace (vm, node, b0, sizeof (ethernet_input_trace_t));
- clib_memcpy_fast (t0->packet_data, b0->data + b0->current_data,
- sizeof (t0->packet_data));
- t0->frame_flags = from_frame->flags;
- clib_memcpy_fast (&t0->frame_data,
- vlib_frame_scalar_args (from_frame),
- sizeof (ethernet_input_frame_t));
+ int classify_filter_result;
+ vlib_buffer_t *b0;
+ bi0 = from[0];
+ from++;
+ n_left--;
+ b0 = vlib_get_buffer (vm, bi0);
+ if (pp->filter_classify_table_index != ~0)
+ {
+ classify_filter_result =
+ vnet_is_packet_traced_inline
+ (b0, pp->filter_classify_table_index, 0 /* full classify */ );
+ if (classify_filter_result)
+ pcap_add_buffer (&pp->pcap_main, vm, bi0,
+ pp->max_bytes_per_pkt);
+ continue;
+ }
+
+ if (pp->pcap_sw_if_index == 0 ||
+ pp->pcap_sw_if_index == vnet_buffer (b0)->sw_if_index[VLIB_RX])
+ {
+ vnet_main_t *vnm = vnet_get_main ();
+ vnet_hw_interface_t *hi =
+ vnet_get_sup_hw_interface
+ (vnm, vnet_buffer (b0)->sw_if_index[VLIB_RX]);
+
+ /* Capture pkt if not filtered, or if filter hits */
+ if (hi->trace_classify_table_index == ~0 ||
+ vnet_is_packet_traced_inline
+ (b0, hi->trace_classify_table_index,
+ 0 /* full classify */ ))
+ pcap_add_buffer (&pp->pcap_main, vm, bi0,
+ pp->max_bytes_per_pkt);
+ }
}
- from += 1;
- n_left -= 1;
}
}
u32 cached_sw_if_index = ~0;
u32 cached_is_l2 = 0; /* shut up gcc */
vnet_hw_interface_t *hi = NULL; /* used for main interface only */
+ vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
+ vlib_buffer_t **b = bufs;
if (variant != ETHERNET_INPUT_VARIANT_ETHERNET)
error_node = vlib_node_get_runtime (vm, ethernet_input_node.index);
next_index = node->cached_next_index;
stats_sw_if_index = node->runtime_data[0];
stats_n_packets = stats_n_bytes = 0;
+ vlib_get_buffers (vm, from, bufs, n_left_from);
while (n_left_from > 0)
{
/* Prefetch next iteration. */
{
- vlib_buffer_t *b2, *b3;
-
- b2 = vlib_get_buffer (vm, from[2]);
- b3 = vlib_get_buffer (vm, from[3]);
-
- vlib_prefetch_buffer_header (b2, STORE);
- vlib_prefetch_buffer_header (b3, STORE);
+ vlib_prefetch_buffer_header (b[2], STORE);
+ vlib_prefetch_buffer_header (b[3], STORE);
- CLIB_PREFETCH (b2->data, sizeof (ethernet_header_t), LOAD);
- CLIB_PREFETCH (b3->data, sizeof (ethernet_header_t), LOAD);
+ CLIB_PREFETCH (b[2]->data, sizeof (ethernet_header_t), LOAD);
+ CLIB_PREFETCH (b[3]->data, sizeof (ethernet_header_t), LOAD);
}
bi0 = from[0];
n_left_to_next -= 2;
n_left_from -= 2;
- b0 = vlib_get_buffer (vm, bi0);
- b1 = vlib_get_buffer (vm, bi1);
+ b0 = b[0];
+ b1 = b[1];
+ b += 2;
error0 = error1 = ETHERNET_ERROR_NONE;
e0 = vlib_buffer_get_current (b0);
// Prefetch next iteration
if (n_left_from > 1)
{
- vlib_buffer_t *p2;
-
- p2 = vlib_get_buffer (vm, from[1]);
- vlib_prefetch_buffer_header (p2, STORE);
- CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
+ vlib_prefetch_buffer_header (b[1], STORE);
+ CLIB_PREFETCH (b[1]->data, CLIB_CACHE_LINE_BYTES, LOAD);
}
bi0 = from[0];
n_left_from -= 1;
n_left_to_next -= 1;
- b0 = vlib_get_buffer (vm, bi0);
+ b0 = b[0];
+ b += 1;
error0 = ETHERNET_ERROR_NONE;
e0 = vlib_buffer_get_current (b0);
return 0;
}
-
-static clib_error_t *
-ethernet_input_init (vlib_main_t * vm)
+void
+ethernet_input_init (vlib_main_t * vm, ethernet_main_t * em)
{
- ethernet_main_t *em = ðernet_main;
__attribute__ ((unused)) vlan_table_t *invalid_vlan_table;
__attribute__ ((unused)) qinq_table_t *invalid_qinq_table;
pool_get (em->vlan_pool, invalid_vlan_table); // first id = 0
// The first qinq pool will always be reserved for an invalid table
pool_get (em->qinq_pool, invalid_qinq_table); // first id = 0
-
- return 0;
}
-VLIB_INIT_FUNCTION (ethernet_input_init);
-
void
ethernet_register_input_type (vlib_main_t * vm,
ethernet_type_t type, u32 node_index)
}
ti = ethernet_get_type_info (em, type);
+ if (ti == 0)
+ {
+ clib_warning ("type_info NULL for type %d", type);
+ return;
+ }
ti->node_index = node_index;
ti->next_index = vlib_node_add_next (vm,
ethernet_input_node.index, node_index);
Code Review / vpp.git / blobdiff