X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fbonding%2Fdevice.c;h=8a78728ccd836fd7e417730cc1c514499ae05b6a;hb=16de39e1a1d922d42b33dcd1b33c3716e3f766f6;hp=a27524089aa1afbe74ddeeb26bb5cd1c36426669;hpb=4f8863b21405d1ab3e067e978a60be72a343358b;p=vpp.git diff --git a/src/vnet/bonding/device.c b/src/vnet/bonding/device.c index a27524089aa..8a78728ccd8 100644 --- a/src/vnet/bonding/device.c +++ b/src/vnet/bonding/device.c @@ -22,6 +22,9 @@ #include #include #include +#include +#include +#include #define foreach_bond_tx_error \ _(NONE, "no error") \ @@ -61,6 +64,7 @@ format_bond_tx_trace (u8 * s, va_list * args) return s; } +#ifndef CLIB_MARCH_VARIANT u8 * format_bond_interface_name (u8 * s, va_list * args) { @@ -72,6 +76,7 @@ format_bond_interface_name (u8 * s, va_list * args) return s; } +#endif static __clib_unused clib_error_t * bond_set_l2_mode_function (vnet_main_t * vnm, @@ -99,18 +104,6 @@ bond_set_l2_mode_function (vnet_main_t * vnm, ethernet_set_rx_redirect (vnm, sif_hw, 1); } } - else if ((bif_hw->l2_if_count == 0) && (l2_if_adjust == -1)) - { - /* Just removed last L2 subinterface on this port */ - vec_foreach (sw_if_index, bif->slaves) - { - sif_hw = vnet_get_sup_hw_interface (vnm, *sw_if_index); - ethernet_set_flags (vnm, sif_hw->hw_if_index, 0); - - /* Allow ip packets to go directly to ip4-input etc */ - ethernet_set_rx_redirect (vnm, sif_hw, 0); - } - } return 0; } @@ -138,63 +131,56 @@ bond_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags) return 0; } -static inline u32 +static_always_inline u32 bond_load_balance_broadcast (vlib_main_t * vm, vlib_node_runtime_t * node, bond_if_t * bif, vlib_buffer_t * b0, uword slave_count) { - vnet_main_t *vnm = vnet_get_main (); + bond_main_t *bm = &bond_main; vlib_buffer_t *c0; int port; - u32 *to_next = 0; u32 sw_if_index; - vlib_frame_t *f; - u16 thread_index = vlib_get_thread_index (); + u16 thread_index = vm->thread_index; + bond_per_thread_data_t *ptd = vec_elt_at_index (bm->per_thread_data, + thread_index); for (port = 1; port < slave_count; port++) { sw_if_index = *vec_elt_at_index (bif->active_slaves, port); - if (bif->per_thread_info[thread_index].frame[port] == 0) - bif->per_thread_info[thread_index].frame[port] = - vnet_get_frame_to_sw_interface (vnm, sw_if_index); - f = bif->per_thread_info[thread_index].frame[port]; - to_next = vlib_frame_vector_args (f); - to_next += f->n_vectors; c0 = vlib_buffer_copy (vm, b0); if (PREDICT_TRUE (c0 != 0)) { vnet_buffer (c0)->sw_if_index[VLIB_TX] = sw_if_index; - to_next[0] = vlib_get_buffer_index (vm, c0); - f->n_vectors++; + ptd->per_port_queue[sw_if_index].buffers[ptd->per_port_queue + [sw_if_index].n_buffers] = + vlib_get_buffer_index (vm, c0); + ptd->per_port_queue[sw_if_index].n_buffers++; } } return 0; } -static inline u32 +static_always_inline u32 bond_load_balance_l2 (vlib_main_t * vm, vlib_node_runtime_t * node, bond_if_t * bif, vlib_buffer_t * b0, uword slave_count) { ethernet_header_t *eth = (ethernet_header_t *) vlib_buffer_get_current (b0); - u32 a = 0, b = 0, c = 0, t1, t2; - u16 t11, t22; - - memcpy (&t1, eth->src_address, sizeof (t1)); - memcpy (&t11, ð->src_address[4], sizeof (t11)); - a = t1 ^ t11; - - memcpy (&t2, eth->dst_address, sizeof (t2)); - memcpy (&t22, ð->dst_address[4], sizeof (t22)); - b = t2 ^ t22; + u32 c; + u64 *dst = (u64 *) & eth->dst_address[0]; + u64 a = clib_mem_unaligned (dst, u64); + u32 *src = (u32 *) & eth->src_address[2]; + u32 b = clib_mem_unaligned (src, u32); - hash_v3_mix32 (a, b, c); - hash_v3_finalize32 (a, b, c); + c = lb_hash_hash_2_tuples (a, b); - return c % slave_count; + if (BOND_MODULO_SHORTCUT (slave_count)) + return (c & (slave_count - 1)); + else + return c % slave_count; } -static inline u16 * +static_always_inline u16 * bond_locate_ethertype (ethernet_header_t * eth) { u16 *ethertype_p; @@ -217,7 +203,7 @@ bond_locate_ethertype (ethernet_header_t * eth) return ethertype_p; } -static inline u32 +static_always_inline u32 bond_load_balance_l23 (vlib_main_t * vm, vlib_node_runtime_t * node, bond_if_t * bif, vlib_buffer_t * b0, uword slave_count) { @@ -225,9 +211,10 @@ bond_load_balance_l23 (vlib_main_t * vm, vlib_node_runtime_t * node, u8 ip_version; ip4_header_t *ip4; u16 ethertype, *ethertype_p; + u32 *mac1, *mac2, *mac3; ethertype_p = bond_locate_ethertype (eth); - ethertype = *ethertype_p; + ethertype = clib_mem_unaligned (ethertype_p, u16); if ((ethertype != htons (ETHERNET_TYPE_IP4)) && (ethertype != htons (ETHERNET_TYPE_IP6))) @@ -238,55 +225,63 @@ bond_load_balance_l23 (vlib_main_t * vm, vlib_node_runtime_t * node, if (ip_version == 0x4) { - u16 t11, t22; - u32 a = 0, b = 0, c = 0, t1, t2; - - memcpy (&t1, eth->src_address, sizeof (t1)); - memcpy (&t11, ð->src_address[4], sizeof (t11)); - a = t1 ^ t11; - - memcpy (&t2, eth->dst_address, sizeof (t2)); - memcpy (&t22, ð->dst_address[4], sizeof (t22)); - b = t2 ^ t22; - - c = ip4->src_address.data_u32 ^ ip4->dst_address.data_u32; - - hash_v3_mix32 (a, b, c); - hash_v3_finalize32 (a, b, c); - - return c % slave_count; + u32 a, c; + + mac1 = (u32 *) & eth->dst_address[0]; + mac2 = (u32 *) & eth->dst_address[4]; + mac3 = (u32 *) & eth->src_address[2]; + + a = clib_mem_unaligned (mac1, u32) ^ clib_mem_unaligned (mac2, u32) ^ + clib_mem_unaligned (mac3, u32); + c = + lb_hash_hash_2_tuples (clib_mem_unaligned (&ip4->address_pair, u64), + a); + if (BOND_MODULO_SHORTCUT (slave_count)) + return (c & (slave_count - 1)); + else + return c % slave_count; } else if (ip_version == 0x6) { - u64 a, b, c; - u64 t1 = 0, t2 = 0; + u64 a; + u32 c; ip6_header_t *ip6 = (ip6_header_t *) (eth + 1); - memcpy (&t1, eth->src_address, sizeof (eth->src_address)); - memcpy (&t2, eth->dst_address, sizeof (eth->dst_address)); - a = t1 ^ t2; - - b = (ip6->src_address.as_u64[0] ^ ip6->src_address.as_u64[1]); - c = (ip6->dst_address.as_u64[0] ^ ip6->dst_address.as_u64[1]); - - hash_mix64 (a, b, c); - return c % slave_count; + mac1 = (u32 *) & eth->dst_address[0]; + mac2 = (u32 *) & eth->dst_address[4]; + mac3 = (u32 *) & eth->src_address[2]; + + a = clib_mem_unaligned (mac1, u32) ^ clib_mem_unaligned (mac2, u32) ^ + clib_mem_unaligned (mac3, u32); + c = + lb_hash_hash (clib_mem_unaligned + (&ip6->src_address.as_uword[0], uword), + clib_mem_unaligned (&ip6->src_address.as_uword[1], + uword), + clib_mem_unaligned (&ip6->dst_address.as_uword[0], + uword), + clib_mem_unaligned (&ip6->dst_address.as_uword[1], + uword), a); + if (BOND_MODULO_SHORTCUT (slave_count)) + return (c & (slave_count - 1)); + else + return c % slave_count; } return (bond_load_balance_l2 (vm, node, bif, b0, slave_count)); } -static inline u32 +static_always_inline u32 bond_load_balance_l34 (vlib_main_t * vm, vlib_node_runtime_t * node, bond_if_t * bif, vlib_buffer_t * b0, uword slave_count) { ethernet_header_t *eth = (ethernet_header_t *) vlib_buffer_get_current (b0); u8 ip_version; - uword is_tcp_udp = 0; + uword is_tcp_udp; ip4_header_t *ip4; u16 ethertype, *ethertype_p; ethertype_p = bond_locate_ethertype (eth); - ethertype = *ethertype_p; + ethertype = clib_mem_unaligned (ethertype_p, u16); if ((ethertype != htons (ETHERNET_TYPE_IP4)) && (ethertype != htons (ETHERNET_TYPE_IP6))) @@ -297,29 +292,30 @@ bond_load_balance_l34 (vlib_main_t * vm, vlib_node_runtime_t * node, if (ip_version == 0x4) { - u32 a = 0, b = 0, c = 0, t1, t2; + u32 a, c, t1, t2; tcp_header_t *tcp = (void *) (ip4 + 1); + is_tcp_udp = (ip4->protocol == IP_PROTOCOL_TCP) || (ip4->protocol == IP_PROTOCOL_UDP); - - a = ip4->src_address.data_u32 ^ ip4->dst_address.data_u32; - - t1 = is_tcp_udp ? tcp->src : 0; - t2 = is_tcp_udp ? tcp->dst : 0; - b = t1 + (t2 << 16); - - hash_v3_mix32 (a, b, c); - hash_v3_finalize32 (a, b, c); - - return c % slave_count; + t1 = is_tcp_udp ? clib_mem_unaligned (&tcp->src, u16) : 0; + t2 = is_tcp_udp ? clib_mem_unaligned (&tcp->dst, u16) : 0; + a = t1 ^ t2; + c = + lb_hash_hash_2_tuples (clib_mem_unaligned (&ip4->address_pair, u64), + a); + if (BOND_MODULO_SHORTCUT (slave_count)) + return (c & (slave_count - 1)); + else + return c % slave_count; } else if (ip_version == 0x6) { - u64 a, b, c; - u64 t1, t2; + u64 a; + u32 c, t1, t2; ip6_header_t *ip6 = (ip6_header_t *) (eth + 1); tcp_header_t *tcp = (void *) (ip6 + 1); + is_tcp_udp = 0; if (PREDICT_TRUE ((ip6->protocol == IP_PROTOCOL_TCP) || (ip6->protocol == IP_PROTOCOL_UDP))) { @@ -337,33 +333,43 @@ bond_load_balance_l34 (vlib_main_t * vm, vlib_node_runtime_t * node, tcp = (tcp_header_t *) ((u8 *) hbh + ((hbh->length + 1) << 3)); } } - a = (ip6->src_address.as_u64[0] ^ ip6->src_address.as_u64[1]); - b = (ip6->dst_address.as_u64[0] ^ ip6->dst_address.as_u64[1]); - - t1 = is_tcp_udp ? tcp->src : 0; - t2 = is_tcp_udp ? tcp->dst : 0; - c = (t2 << 16) | t1; - hash_mix64 (a, b, c); - - return c % slave_count; + t1 = is_tcp_udp ? clib_mem_unaligned (&tcp->src, u16) : 0; + t2 = is_tcp_udp ? clib_mem_unaligned (&tcp->dst, u16) : 0; + a = t1 ^ t2; + c = + lb_hash_hash (clib_mem_unaligned + (&ip6->src_address.as_uword[0], uword), + clib_mem_unaligned (&ip6->src_address.as_uword[1], + uword), + clib_mem_unaligned (&ip6->dst_address.as_uword[0], + uword), + clib_mem_unaligned (&ip6->dst_address.as_uword[1], + uword), a); + if (BOND_MODULO_SHORTCUT (slave_count)) + return (c & (slave_count - 1)); + else + return c % slave_count; } return (bond_load_balance_l2 (vm, node, bif, b0, slave_count)); } -static inline u32 +static_always_inline u32 bond_load_balance_round_robin (vlib_main_t * vm, vlib_node_runtime_t * node, bond_if_t * bif, vlib_buffer_t * b0, uword slave_count) { bif->lb_rr_last_index++; - bif->lb_rr_last_index %= slave_count; + if (BOND_MODULO_SHORTCUT (slave_count)) + bif->lb_rr_last_index &= slave_count - 1; + else + bif->lb_rr_last_index %= slave_count; return bif->lb_rr_last_index; } -static inline u32 +static_always_inline u32 bond_load_balance_active_backup (vlib_main_t * vm, vlib_node_runtime_t * node, bond_if_t * bif, vlib_buffer_t * b0, @@ -373,312 +379,401 @@ bond_load_balance_active_backup (vlib_main_t * vm, return 0; } -static bond_load_balance_func_t bond_load_balance_table[] = { -#define _(v,f,s, p) { bond_load_balance_##p }, - foreach_bond_lb_algo -#undef _ -}; - -static uword -bond_tx_fn (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * frame) +static_always_inline void +bond_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * frame, bond_if_t * bif, + uword slave_count, u32 lb_alg) { - vnet_interface_output_runtime_t *rund = (void *) node->runtime_data; bond_main_t *bm = &bond_main; - bond_if_t *bif = pool_elt_at_index (bm->interfaces, rund->dev_instance); - u32 bi0, bi1, bi2, bi3; - vlib_buffer_t *b0, *b1, *b2, *b3; - u32 *from = vlib_frame_vector_args (frame); - u32 n_left_from; - ethernet_header_t *eth; - u32 next0 = 0, next1 = 0, next2 = 0, next3 = 0; - u32 port, port1, port2, port3; - u32 sw_if_index, sw_if_index1, sw_if_index2, sw_if_index3; + vnet_main_t *vnm = vnet_get_main (); + u16 thread_index = vm->thread_index; bond_packet_trace_t *t0; uword n_trace = vlib_get_trace_count (vm, node); - u16 thread_index = vlib_get_thread_index (); - vnet_main_t *vnm = vnet_get_main (); u32 *to_next; - u32 sif_if_index, sif_if_index1, sif_if_index2, sif_if_index3; vlib_frame_t *f; - uword slave_count; + ethernet_header_t *eth; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; + u32 *from = vlib_frame_vector_args (frame); + u32 n_left = frame->n_vectors; + u32 sw_if_index; + u32 port0 = 0, port1 = 0, port2 = 0, port3 = 0; + bond_per_thread_data_t *ptd = vec_elt_at_index (bm->per_thread_data, + thread_index); - if (PREDICT_FALSE (bif->admin_up == 0)) + vlib_get_buffers (vm, from, bufs, n_left); + b = bufs; + while (n_left >= 4) { - vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors); - vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters + - VNET_INTERFACE_COUNTER_DROP, - thread_index, bif->sw_if_index, - frame->n_vectors); - vlib_error_count (vm, node->node_index, BOND_TX_ERROR_IF_DOWN, - frame->n_vectors); - return frame->n_vectors; - } + u32 sif_if_index0, sif_if_index1, sif_if_index2, sif_if_index3; - clib_spinlock_lock_if_init (&bif->lockp); - slave_count = vec_len (bif->active_slaves); - if (PREDICT_FALSE (slave_count == 0)) - { - bi0 = from[0]; - b0 = vlib_get_buffer (vm, bi0); - vlib_increment_combined_counter - (vnet_main.interface_main.combined_sw_if_counters - + VNET_INTERFACE_COUNTER_TX, thread_index, bif->sw_if_index, - frame->n_vectors, b0->current_length); + // Prefetch next iteration + if (n_left >= 8) + { + vlib_buffer_t **pb = b + 4; - vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors); - vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters + - VNET_INTERFACE_COUNTER_DROP, - thread_index, bif->sw_if_index, - frame->n_vectors); - vlib_error_count (vm, node->node_index, BOND_TX_ERROR_NO_SLAVE, - frame->n_vectors); - clib_spinlock_unlock_if_init (&bif->lockp); - return frame->n_vectors; - } + vlib_prefetch_buffer_header (pb[0], LOAD); + vlib_prefetch_buffer_header (pb[1], LOAD); + vlib_prefetch_buffer_header (pb[2], LOAD); + vlib_prefetch_buffer_header (pb[3], LOAD); - vec_validate_aligned (bif->per_thread_info[thread_index].frame, slave_count, - CLIB_CACHE_LINE_BYTES); + CLIB_PREFETCH (pb[0]->data, CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (pb[1]->data, CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (pb[2]->data, CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (pb[3]->data, CLIB_CACHE_LINE_BYTES, LOAD); + } - /* Number of buffers / pkts */ - n_left_from = frame->n_vectors; + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]); - while (n_left_from > 0) - { - while (n_left_from >= 4) + if (PREDICT_TRUE (slave_count > 1)) { - // Prefetch next iteration - if (n_left_from >= 8) + if (lb_alg == BOND_LB_L2) { - vlib_buffer_t *p4, *p5, *p6, *p7; - - p4 = vlib_get_buffer (vm, from[4]); - p5 = vlib_get_buffer (vm, from[5]); - p6 = vlib_get_buffer (vm, from[6]); - p7 = vlib_get_buffer (vm, from[7]); - - vlib_prefetch_buffer_header (p4, STORE); - vlib_prefetch_buffer_header (p5, STORE); - vlib_prefetch_buffer_header (p6, STORE); - vlib_prefetch_buffer_header (p7, STORE); - - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, LOAD); + port0 = bond_load_balance_l2 (vm, node, bif, b[0], slave_count); + port1 = bond_load_balance_l2 (vm, node, bif, b[1], slave_count); + port2 = bond_load_balance_l2 (vm, node, bif, b[2], slave_count); + port3 = bond_load_balance_l2 (vm, node, bif, b[3], slave_count); } + else if (lb_alg == BOND_LB_L34) + { + port0 = bond_load_balance_l34 (vm, node, bif, b[0], + slave_count); + port1 = bond_load_balance_l34 (vm, node, bif, b[1], + slave_count); + port2 = bond_load_balance_l34 (vm, node, bif, b[2], + slave_count); + port3 = bond_load_balance_l34 (vm, node, bif, b[3], + slave_count); + } + else if (lb_alg == BOND_LB_L23) + { + port0 = bond_load_balance_l23 (vm, node, bif, b[0], + slave_count); + port1 = bond_load_balance_l23 (vm, node, bif, b[1], + slave_count); + port2 = bond_load_balance_l23 (vm, node, bif, b[2], + slave_count); + port3 = bond_load_balance_l23 (vm, node, bif, b[3], + slave_count); + } + else if (lb_alg == BOND_LB_RR) + { + port0 = bond_load_balance_round_robin (vm, node, bif, b[0], + slave_count); + port1 = bond_load_balance_round_robin (vm, node, bif, b[1], + slave_count); + port2 = bond_load_balance_round_robin (vm, node, bif, b[2], + slave_count); + port3 = bond_load_balance_round_robin (vm, node, bif, b[3], + slave_count); + } + else if (lb_alg == BOND_LB_BC) + { + port0 = bond_load_balance_broadcast (vm, node, bif, b[0], + slave_count); + port1 = bond_load_balance_broadcast (vm, node, bif, b[1], + slave_count); + port2 = bond_load_balance_broadcast (vm, node, bif, b[2], + slave_count); + port3 = bond_load_balance_broadcast (vm, node, bif, b[3], + slave_count); + } + else if (lb_alg == BOND_LB_AB) + { + port0 = bond_load_balance_active_backup (vm, node, bif, b[0], + slave_count); + port1 = bond_load_balance_active_backup (vm, node, bif, b[1], + slave_count); + port2 = bond_load_balance_active_backup (vm, node, bif, b[2], + slave_count); + port3 = bond_load_balance_active_backup (vm, node, bif, b[3], + slave_count); + } + else + { + ASSERT (0); + } + } - bi0 = from[0]; - bi1 = from[1]; - bi2 = from[2]; - bi3 = from[3]; - - b0 = vlib_get_buffer (vm, bi0); - b1 = vlib_get_buffer (vm, bi1); - b2 = vlib_get_buffer (vm, bi2); - b3 = vlib_get_buffer (vm, bi3); - - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3); - - sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX]; - sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_TX]; - sw_if_index2 = vnet_buffer (b2)->sw_if_index[VLIB_TX]; - sw_if_index3 = vnet_buffer (b3)->sw_if_index[VLIB_TX]; - - port = - (bond_load_balance_table[bif->lb]).load_balance (vm, node, bif, - b0, slave_count); - port1 = - (bond_load_balance_table[bif->lb]).load_balance (vm, node, bif, - b1, slave_count); - port2 = - (bond_load_balance_table[bif->lb]).load_balance (vm, node, bif, - b2, slave_count); - port3 = - (bond_load_balance_table[bif->lb]).load_balance (vm, node, bif, - b3, slave_count); - - sif_if_index = *vec_elt_at_index (bif->active_slaves, port); - sif_if_index1 = *vec_elt_at_index (bif->active_slaves, port1); - sif_if_index2 = *vec_elt_at_index (bif->active_slaves, port2); - sif_if_index3 = *vec_elt_at_index (bif->active_slaves, port3); - - vnet_buffer (b0)->sw_if_index[VLIB_TX] = sif_if_index; - vnet_buffer (b1)->sw_if_index[VLIB_TX] = sif_if_index1; - vnet_buffer (b2)->sw_if_index[VLIB_TX] = sif_if_index2; - vnet_buffer (b3)->sw_if_index[VLIB_TX] = sif_if_index3; - - if (bif->per_thread_info[thread_index].frame[port] == 0) - bif->per_thread_info[thread_index].frame[port] = - vnet_get_frame_to_sw_interface (vnm, sif_if_index); - - if (bif->per_thread_info[thread_index].frame[port1] == 0) - bif->per_thread_info[thread_index].frame[port1] = - vnet_get_frame_to_sw_interface (vnm, sif_if_index1); - - if (bif->per_thread_info[thread_index].frame[port2] == 0) - bif->per_thread_info[thread_index].frame[port2] = - vnet_get_frame_to_sw_interface (vnm, sif_if_index2); - - if (bif->per_thread_info[thread_index].frame[port3] == 0) - bif->per_thread_info[thread_index].frame[port3] = - vnet_get_frame_to_sw_interface (vnm, sif_if_index3); - - f = bif->per_thread_info[thread_index].frame[port]; - to_next = vlib_frame_vector_args (f); - to_next += f->n_vectors; - to_next[0] = vlib_get_buffer_index (vm, b0); - f->n_vectors++; - - f = bif->per_thread_info[thread_index].frame[port1]; - to_next = vlib_frame_vector_args (f); - to_next += f->n_vectors; - to_next[0] = vlib_get_buffer_index (vm, b1); - f->n_vectors++; - - f = bif->per_thread_info[thread_index].frame[port2]; - to_next = vlib_frame_vector_args (f); - to_next += f->n_vectors; - to_next[0] = vlib_get_buffer_index (vm, b2); - f->n_vectors++; - - f = bif->per_thread_info[thread_index].frame[port3]; - to_next = vlib_frame_vector_args (f); - to_next += f->n_vectors; - to_next[0] = vlib_get_buffer_index (vm, b3); - f->n_vectors++; + sif_if_index0 = *vec_elt_at_index (bif->active_slaves, port0); + sif_if_index1 = *vec_elt_at_index (bif->active_slaves, port1); + sif_if_index2 = *vec_elt_at_index (bif->active_slaves, port2); + sif_if_index3 = *vec_elt_at_index (bif->active_slaves, port3); - if (PREDICT_FALSE (n_trace > 0)) + /* Do the tracing before the interface is overwritten */ + if (PREDICT_FALSE (n_trace > 0)) + { + u32 next0 = 0, next1 = 0, next2 = 0, next3 = 0; + vlib_trace_buffer (vm, node, next0, b[0], 0 /* follow_chain */ ); + vlib_set_trace_count (vm, node, --n_trace); + t0 = vlib_add_trace (vm, node, b[0], sizeof (*t0)); + eth = (ethernet_header_t *) vlib_buffer_get_current (b[0]); + t0->ethernet = *eth; + t0->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; + t0->bond_sw_if_index = sif_if_index0; + + if (PREDICT_TRUE (n_trace > 0)) { - vlib_trace_buffer (vm, node, next0, b0, 0 /* follow_chain */ ); + vlib_trace_buffer (vm, node, next1, b[1], + 0 /* follow_chain */ ); vlib_set_trace_count (vm, node, --n_trace); - t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); - eth = (ethernet_header_t *) vlib_buffer_get_current (b0); + t0 = vlib_add_trace (vm, node, b[1], sizeof (*t0)); + eth = (ethernet_header_t *) vlib_buffer_get_current (b[1]); t0->ethernet = *eth; - t0->sw_if_index = sw_if_index; - t0->bond_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX]; + t0->sw_if_index = vnet_buffer (b[1])->sw_if_index[VLIB_TX]; + t0->bond_sw_if_index = sif_if_index1; if (PREDICT_TRUE (n_trace > 0)) { - vlib_trace_buffer (vm, node, next1, b1, + vlib_trace_buffer (vm, node, next2, b[2], 0 /* follow_chain */ ); vlib_set_trace_count (vm, node, --n_trace); - t0 = vlib_add_trace (vm, node, b1, sizeof (*t0)); - eth = (ethernet_header_t *) vlib_buffer_get_current (b1); + t0 = vlib_add_trace (vm, node, b[2], sizeof (*t0)); + eth = (ethernet_header_t *) vlib_buffer_get_current (b[2]); t0->ethernet = *eth; - t0->sw_if_index = sw_if_index1; - t0->bond_sw_if_index = - vnet_buffer (b1)->sw_if_index[VLIB_TX]; + t0->sw_if_index = vnet_buffer (b[2])->sw_if_index[VLIB_TX]; + t0->bond_sw_if_index = sif_if_index2; if (PREDICT_TRUE (n_trace > 0)) { - vlib_trace_buffer (vm, node, next2, b2, + vlib_trace_buffer (vm, node, next3, b[3], 0 /* follow_chain */ ); vlib_set_trace_count (vm, node, --n_trace); - t0 = vlib_add_trace (vm, node, b2, sizeof (*t0)); + t0 = vlib_add_trace (vm, node, b[3], sizeof (*t0)); eth = - (ethernet_header_t *) vlib_buffer_get_current (b2); + (ethernet_header_t *) vlib_buffer_get_current (b[3]); t0->ethernet = *eth; - t0->sw_if_index = sw_if_index2; - t0->bond_sw_if_index = - vnet_buffer (b2)->sw_if_index[VLIB_TX]; - - if (PREDICT_TRUE (n_trace > 0)) - { - vlib_trace_buffer (vm, node, next3, b3, - 0 /* follow_chain */ ); - vlib_set_trace_count (vm, node, --n_trace); - t0 = vlib_add_trace (vm, node, b3, sizeof (*t0)); - eth = - (ethernet_header_t *) - vlib_buffer_get_current (b3); - t0->ethernet = *eth; - t0->sw_if_index = sw_if_index3; - t0->bond_sw_if_index = - vnet_buffer (b3)->sw_if_index[VLIB_TX]; - } + t0->sw_if_index = + vnet_buffer (b[3])->sw_if_index[VLIB_TX]; + t0->bond_sw_if_index = sif_if_index3; } } } - from += 4; - n_left_from -= 4; } - while (n_left_from > 0) + vnet_buffer (b[0])->sw_if_index[VLIB_TX] = sif_if_index0; + vnet_buffer (b[1])->sw_if_index[VLIB_TX] = sif_if_index1; + vnet_buffer (b[2])->sw_if_index[VLIB_TX] = sif_if_index2; + vnet_buffer (b[3])->sw_if_index[VLIB_TX] = sif_if_index3; + + ptd->per_port_queue[sif_if_index0].buffers[ptd->per_port_queue + [sif_if_index0].n_buffers] = + vlib_get_buffer_index (vm, b[0]); + ptd->per_port_queue[sif_if_index0].n_buffers++; + + ptd->per_port_queue[sif_if_index1].buffers[ptd->per_port_queue + [sif_if_index1].n_buffers] = + vlib_get_buffer_index (vm, b[1]); + ptd->per_port_queue[sif_if_index1].n_buffers++; + + ptd->per_port_queue[sif_if_index2].buffers[ptd->per_port_queue + [sif_if_index2].n_buffers] = + vlib_get_buffer_index (vm, b[2]); + ptd->per_port_queue[sif_if_index2].n_buffers++; + + ptd->per_port_queue[sif_if_index3].buffers[ptd->per_port_queue + [sif_if_index3].n_buffers] = + vlib_get_buffer_index (vm, b[3]); + ptd->per_port_queue[sif_if_index3].n_buffers++; + + n_left -= 4; + b += 4; + } + + while (n_left > 0) + { + u32 sif_if_index0; + + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); + + if (PREDICT_TRUE (slave_count > 1)) { - // Prefetch next iteration - if (n_left_from > 1) + if (bif->lb == BOND_LB_L2) { - vlib_buffer_t *p2; - - p2 = vlib_get_buffer (vm, from[1]); - vlib_prefetch_buffer_header (p2, STORE); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); + port0 = bond_load_balance_l2 (vm, node, bif, b[0], slave_count); + } + else if (bif->lb == BOND_LB_L34) + { + port0 = bond_load_balance_l34 (vm, node, bif, b[0], + slave_count); } + else if (bif->lb == BOND_LB_L23) + { + port0 = bond_load_balance_l23 (vm, node, bif, b[0], + slave_count); + } + else if (bif->lb == BOND_LB_RR) + { + port0 = bond_load_balance_round_robin (vm, node, bif, b[0], + slave_count); + } + else if (bif->lb == BOND_LB_BC) + { + port0 = bond_load_balance_broadcast (vm, node, bif, b[0], + slave_count); + } + else if (bif->lb == BOND_LB_AB) + { + port0 = bond_load_balance_active_backup (vm, node, bif, b[0], + slave_count); + } + else + { + ASSERT (0); + } + } - bi0 = from[0]; - b0 = vlib_get_buffer (vm, bi0); + sif_if_index0 = *vec_elt_at_index (bif->active_slaves, port0); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); + /* Do the tracing before the old interface is overwritten */ + if (PREDICT_FALSE (n_trace > 0)) + { + u32 next0 = 0; + + vlib_trace_buffer (vm, node, next0, b[0], 0 /* follow_chain */ ); + vlib_set_trace_count (vm, node, --n_trace); + t0 = vlib_add_trace (vm, node, b[0], sizeof (*t0)); + eth = (ethernet_header_t *) vlib_buffer_get_current (b[0]); + t0->ethernet = *eth; + t0->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; + t0->bond_sw_if_index = sif_if_index0; + } - sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX]; + vnet_buffer (b[0])->sw_if_index[VLIB_TX] = sif_if_index0; - port = - (bond_load_balance_table[bif->lb]).load_balance (vm, node, bif, - b0, slave_count); - sif_if_index = *vec_elt_at_index (bif->active_slaves, port); - vnet_buffer (b0)->sw_if_index[VLIB_TX] = sif_if_index; - if (bif->per_thread_info[thread_index].frame[port] == 0) - bif->per_thread_info[thread_index].frame[port] = - vnet_get_frame_to_sw_interface (vnm, sif_if_index); - f = bif->per_thread_info[thread_index].frame[port]; - to_next = vlib_frame_vector_args (f); - to_next += f->n_vectors; - to_next[0] = vlib_get_buffer_index (vm, b0); - f->n_vectors++; + ptd->per_port_queue[sif_if_index0].buffers[ptd->per_port_queue + [sif_if_index0].n_buffers] = + vlib_get_buffer_index (vm, b[0]); + ptd->per_port_queue[sif_if_index0].n_buffers++; - if (PREDICT_FALSE (n_trace > 0)) - { - vlib_trace_buffer (vm, node, next0, b0, 0 /* follow_chain */ ); - vlib_set_trace_count (vm, node, --n_trace); - t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); - eth = (ethernet_header_t *) vlib_buffer_get_current (b0); - t0->ethernet = *eth; - t0->sw_if_index = sw_if_index; - t0->bond_sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX]; - } - - from += 1; - n_left_from -= 1; - } + n_left -= 1; + b += 1; } - for (port = 0; port < slave_count; port++) + for (port0 = 0; port0 < slave_count; port0++) { - f = bif->per_thread_info[thread_index].frame[port]; - if (f == 0) - continue; - - sw_if_index = *vec_elt_at_index (bif->active_slaves, port); - vnet_put_frame_to_sw_interface (vnm, sw_if_index, f); - bif->per_thread_info[thread_index].frame[port] = 0; + sw_if_index = *vec_elt_at_index (bif->active_slaves, port0); + if (PREDICT_TRUE (ptd->per_port_queue[sw_if_index].n_buffers)) + { + f = vnet_get_frame_to_sw_interface (vnm, sw_if_index); + f->n_vectors = ptd->per_port_queue[sw_if_index].n_buffers; + to_next = vlib_frame_vector_args (f); + clib_memcpy (to_next, ptd->per_port_queue[sw_if_index].buffers, + f->n_vectors << 2); + vnet_put_frame_to_sw_interface (vnm, sw_if_index, f); + ptd->per_port_queue[sw_if_index].n_buffers = 0; + } } vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, bif->sw_if_index, frame->n_vectors); +} + +VNET_DEVICE_CLASS_TX_FN (bond_dev_class) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + vnet_interface_output_runtime_t *rund = (void *) node->runtime_data; + bond_main_t *bm = &bond_main; + u16 thread_index = vm->thread_index; + bond_if_t *bif = pool_elt_at_index (bm->interfaces, rund->dev_instance); + uword slave_count; + + if (PREDICT_FALSE (bif->admin_up == 0)) + { + vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors); + vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters + + VNET_INTERFACE_COUNTER_DROP, + thread_index, bif->sw_if_index, + frame->n_vectors); + vlib_error_count (vm, node->node_index, BOND_TX_ERROR_IF_DOWN, + frame->n_vectors); + return frame->n_vectors; + } + + slave_count = vec_len (bif->active_slaves); + if (PREDICT_FALSE (slave_count == 0)) + { + vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors); + vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters + + VNET_INTERFACE_COUNTER_DROP, + thread_index, bif->sw_if_index, + frame->n_vectors); + vlib_error_count (vm, node->node_index, BOND_TX_ERROR_NO_SLAVE, + frame->n_vectors); + return frame->n_vectors; + } + + if (bif->lb == BOND_LB_L2) + bond_tx_inline (vm, node, frame, bif, slave_count, BOND_LB_L2); + else if (bif->lb == BOND_LB_L34) + bond_tx_inline (vm, node, frame, bif, slave_count, BOND_LB_L34); + else if (bif->lb == BOND_LB_L23) + bond_tx_inline (vm, node, frame, bif, slave_count, BOND_LB_L23); + else if (bif->lb == BOND_LB_RR) + bond_tx_inline (vm, node, frame, bif, slave_count, BOND_LB_RR); + else if (bif->lb == BOND_LB_BC) + bond_tx_inline (vm, node, frame, bif, slave_count, BOND_LB_BC); + else if (bif->lb == BOND_LB_AB) + bond_tx_inline (vm, node, frame, bif, slave_count, BOND_LB_AB); + else + ASSERT (0); - clib_spinlock_unlock_if_init (&bif->lockp); return frame->n_vectors; } +static walk_rc_t +bond_active_interface_switch_cb (vnet_main_t * vnm, u32 sw_if_index, + void *arg) +{ + bond_main_t *bm = &bond_main; + + send_ip4_garp (bm->vlib_main, sw_if_index); + send_ip6_na (bm->vlib_main, sw_if_index); + + return (WALK_CONTINUE); +} + +static uword +bond_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) +{ + vnet_main_t *vnm = vnet_get_main (); + uword event_type, *event_data = 0; + + while (1) + { + u32 i; + u32 hw_if_index; + + vlib_process_wait_for_event (vm); + event_type = vlib_process_get_events (vm, &event_data); + ASSERT (event_type == BOND_SEND_GARP_NA); + for (i = 0; i < vec_len (event_data); i++) + { + hw_if_index = event_data[i]; + /* walk hw interface to process all subinterfaces */ + vnet_hw_interface_walk_sw (vnm, hw_if_index, + bond_active_interface_switch_cb, 0); + } + vec_reset_length (event_data); + } + return 0; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (bond_process_node) = { + .function = bond_process, + .type = VLIB_NODE_TYPE_PROCESS, + .name = "bond-process", +}; +/* *INDENT-ON* */ + /* *INDENT-OFF* */ VNET_DEVICE_CLASS (bond_dev_class) = { .name = "bond", - .tx_function = bond_tx_fn, .tx_function_n_errors = BOND_TX_N_ERROR, .tx_function_error_strings = bond_tx_error_strings, .format_device_name = format_bond_interface_name, @@ -688,7 +783,6 @@ VNET_DEVICE_CLASS (bond_dev_class) = { .format_tx_trace = format_bond_tx_trace, }; -VLIB_DEVICE_TX_FUNCTION_MULTIARCH (bond_dev_class, bond_tx_fn) /* *INDENT-ON* */ /*