X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fbonding%2Fdevice.c;h=38759eed1ef41b1e904a6a738a0f65cb6244496b;hb=4c4223edf;hp=7f9828407fbe87284bb6642b3ee02dd00d728a5e;hpb=cefe1341c6a536ebe161d97ab6fb10db95e57d05;p=vpp.git diff --git a/src/vnet/bonding/device.c b/src/vnet/bonding/device.c index 7f9828407fb..38759eed1ef 100644 --- a/src/vnet/bonding/device.c +++ b/src/vnet/bonding/device.c @@ -24,12 +24,12 @@ #include #include #include -#include +#include #define foreach_bond_tx_error \ _(NONE, "no error") \ _(IF_DOWN, "interface down") \ - _(NO_SLAVE, "no slave") + _(NO_MEMBER, "no member") typedef enum { @@ -72,7 +72,7 @@ format_bond_interface_name (u8 * s, va_list * args) bond_main_t *bm = &bond_main; bond_if_t *bif = pool_elt_at_index (bm->interfaces, dev_instance); - s = format (s, "BondEthernet%lu", bif->dev_instance); + s = format (s, "BondEthernet%lu", bif->id); return s; } @@ -85,23 +85,30 @@ bond_set_l2_mode_function (vnet_main_t * vnm, { bond_if_t *bif; u32 *sw_if_index; - struct vnet_hw_interface_t *sif_hw; + struct vnet_hw_interface_t *mif_hw; - bif = bond_get_master_by_sw_if_index (bif_hw->sw_if_index); + bif = bond_get_bond_if_by_sw_if_index (bif_hw->sw_if_index); if (!bif) return 0; if ((bif_hw->l2_if_count == 1) && (l2_if_adjust == 1)) { /* Just added first L2 interface on this port */ - vec_foreach (sw_if_index, bif->slaves) + vec_foreach (sw_if_index, bif->members) { - sif_hw = vnet_get_sup_hw_interface (vnm, *sw_if_index); - ethernet_set_flags (vnm, sif_hw->hw_if_index, + mif_hw = vnet_get_sup_hw_interface (vnm, *sw_if_index); + ethernet_set_flags (vnm, mif_hw->hw_if_index, ETHERNET_INTERFACE_FLAG_ACCEPT_ALL); - - /* ensure all packets go to ethernet-input */ - ethernet_set_rx_redirect (vnm, sif_hw, 1); + } + } + else if ((bif_hw->l2_if_count == 0) && (l2_if_adjust == -1)) + { + /* Just removed last L2 subinterface on this port */ + vec_foreach (sw_if_index, bif->members) + { + mif_hw = vnet_get_sup_hw_interface (vnm, *sw_if_index); + ethernet_set_flags (vnm, mif_hw->hw_if_index, + /*ETHERNET_INTERFACE_FLAG_DEFAULT_L3 */ 0); } } @@ -125,40 +132,84 @@ bond_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags) bond_if_t *bif = pool_elt_at_index (bm->interfaces, hif->dev_instance); bif->admin_up = is_up; - if (is_up && vec_len (bif->active_slaves)) + if (is_up && vec_len (bif->active_members)) vnet_hw_interface_set_flags (vnm, bif->hw_if_index, VNET_HW_INTERFACE_FLAG_LINK_UP); return 0; } -static_always_inline u32 -bond_load_balance_broadcast (vlib_main_t * vm, vlib_node_runtime_t * node, - bond_if_t * bif, vlib_buffer_t * b0, - uword slave_count) +static clib_error_t * +bond_add_del_mac_address (vnet_hw_interface_t * hi, const u8 * address, + u8 is_add) { vnet_main_t *vnm = vnet_get_main (); + bond_if_t *bif; + clib_error_t *error = 0; + vnet_hw_interface_t *s_hi; + int i; + + + bif = bond_get_bond_if_by_sw_if_index (hi->sw_if_index); + if (!bif) + { + return clib_error_return (0, + "No bond interface found for sw_if_index %u", + hi->sw_if_index); + } + + /* Add/del address on each member hw intf, they control the hardware */ + vec_foreach_index (i, bif->members) + { + s_hi = vnet_get_sup_hw_interface (vnm, vec_elt (bif->members, i)); + error = vnet_hw_interface_add_del_mac_address (vnm, s_hi->hw_if_index, + address, is_add); + + if (error) + { + int j; + + /* undo any that were completed before the failure */ + for (j = i - 1; j > -1; j--) + { + s_hi = vnet_get_sup_hw_interface (vnm, vec_elt (bif->members, j)); + vnet_hw_interface_add_del_mac_address (vnm, s_hi->hw_if_index, + address, !(is_add)); + } + + return error; + } + } + + return 0; +} + +static_always_inline void +bond_tx_add_to_queue (bond_per_thread_data_t * ptd, u32 port, u32 bi) +{ + u32 idx = ptd->per_port_queue[port].n_buffers++; + ptd->per_port_queue[port].buffers[idx] = bi; +} + +static_always_inline u32 +bond_lb_broadcast (vlib_main_t * vm, + bond_if_t * bif, vlib_buffer_t * b0, uword n_members) +{ + bond_main_t *bm = &bond_main; vlib_buffer_t *c0; int port; - u32 *to_next = 0; u32 sw_if_index; - vlib_frame_t *f; u16 thread_index = vm->thread_index; + bond_per_thread_data_t *ptd = vec_elt_at_index (bm->per_thread_data, + thread_index); - for (port = 1; port < slave_count; port++) + for (port = 1; port < n_members; port++) { - sw_if_index = *vec_elt_at_index (bif->active_slaves, port); - if (bif->per_thread_info[thread_index].frame[port] == 0) - bif->per_thread_info[thread_index].frame[port] = - vnet_get_frame_to_sw_interface (vnm, sw_if_index); - f = bif->per_thread_info[thread_index].frame[port]; - to_next = vlib_frame_vector_args (f); - to_next += f->n_vectors; + sw_if_index = *vec_elt_at_index (bif->active_members, port); c0 = vlib_buffer_copy (vm, b0); if (PREDICT_TRUE (c0 != 0)) { vnet_buffer (c0)->sw_if_index[VLIB_TX] = sw_if_index; - to_next[0] = vlib_get_buffer_index (vm, c0); - f->n_vectors++; + bond_tx_add_to_queue (ptd, port, vlib_get_buffer_index (vm, c0)); } } @@ -166,22 +217,15 @@ bond_load_balance_broadcast (vlib_main_t * vm, vlib_node_runtime_t * node, } static_always_inline u32 -bond_load_balance_l2 (vlib_main_t * vm, vlib_node_runtime_t * node, - bond_if_t * bif, vlib_buffer_t * b0, uword slave_count) +bond_lb_l2 (vlib_buffer_t * b0) { - ethernet_header_t *eth = (ethernet_header_t *) vlib_buffer_get_current (b0); - u32 c; + ethernet_header_t *eth = vlib_buffer_get_current (b0); u64 *dst = (u64 *) & eth->dst_address[0]; u64 a = clib_mem_unaligned (dst, u64); u32 *src = (u32 *) & eth->src_address[2]; u32 b = clib_mem_unaligned (src, u32); - c = lb_hash_hash_2_tuples (a, b); - - if (BOND_MODULO_SHORTCUT (slave_count)) - return (c & (slave_count - 1)); - else - return c % slave_count; + return lb_hash_hash_2_tuples (a, b); } static_always_inline u16 * @@ -208,10 +252,9 @@ bond_locate_ethertype (ethernet_header_t * eth) } static_always_inline u32 -bond_load_balance_l23 (vlib_main_t * vm, vlib_node_runtime_t * node, - bond_if_t * bif, vlib_buffer_t * b0, uword slave_count) +bond_lb_l23 (vlib_buffer_t * b0) { - ethernet_header_t *eth = (ethernet_header_t *) vlib_buffer_get_current (b0); + ethernet_header_t *eth = vlib_buffer_get_current (b0); u8 ip_version; ip4_header_t *ip4; u16 ethertype, *ethertype_p; @@ -222,7 +265,7 @@ bond_load_balance_l23 (vlib_main_t * vm, vlib_node_runtime_t * node, if ((ethertype != htons (ETHERNET_TYPE_IP4)) && (ethertype != htons (ETHERNET_TYPE_IP6))) - return (bond_load_balance_l2 (vm, node, bif, b0, slave_count)); + return bond_lb_l2 (b0); ip4 = (ip4_header_t *) (ethertype_p + 1); ip_version = (ip4->ip_version_and_header_length >> 4); @@ -240,10 +283,7 @@ bond_load_balance_l23 (vlib_main_t * vm, vlib_node_runtime_t * node, c = lb_hash_hash_2_tuples (clib_mem_unaligned (&ip4->address_pair, u64), a); - if (BOND_MODULO_SHORTCUT (slave_count)) - return (c & (slave_count - 1)); - else - return c % slave_count; + return c; } else if (ip_version == 0x6) { @@ -266,19 +306,15 @@ bond_load_balance_l23 (vlib_main_t * vm, vlib_node_runtime_t * node, uword), clib_mem_unaligned (&ip6->dst_address.as_uword[1], uword), a); - if (BOND_MODULO_SHORTCUT (slave_count)) - return (c & (slave_count - 1)); - else - return c % slave_count; + return c; } - return (bond_load_balance_l2 (vm, node, bif, b0, slave_count)); + return bond_lb_l2 (b0); } static_always_inline u32 -bond_load_balance_l34 (vlib_main_t * vm, vlib_node_runtime_t * node, - bond_if_t * bif, vlib_buffer_t * b0, uword slave_count) +bond_lb_l34 (vlib_buffer_t * b0) { - ethernet_header_t *eth = (ethernet_header_t *) vlib_buffer_get_current (b0); + ethernet_header_t *eth = vlib_buffer_get_current (b0); u8 ip_version; uword is_tcp_udp; ip4_header_t *ip4; @@ -289,14 +325,14 @@ bond_load_balance_l34 (vlib_main_t * vm, vlib_node_runtime_t * node, if ((ethertype != htons (ETHERNET_TYPE_IP4)) && (ethertype != htons (ETHERNET_TYPE_IP6))) - return (bond_load_balance_l2 (vm, node, bif, b0, slave_count)); + return (bond_lb_l2 (b0)); ip4 = (ip4_header_t *) (ethertype_p + 1); ip_version = (ip4->ip_version_and_header_length >> 4); if (ip_version == 0x4) { - u32 a, c, t1, t2; + u32 a, t1, t2; tcp_header_t *tcp = (void *) (ip4 + 1); is_tcp_udp = (ip4->protocol == IP_PROTOCOL_TCP) || @@ -304,13 +340,9 @@ bond_load_balance_l34 (vlib_main_t * vm, vlib_node_runtime_t * node, t1 = is_tcp_udp ? clib_mem_unaligned (&tcp->src, u16) : 0; t2 = is_tcp_udp ? clib_mem_unaligned (&tcp->dst, u16) : 0; a = t1 ^ t2; - c = + return lb_hash_hash_2_tuples (clib_mem_unaligned (&ip4->address_pair, u64), a); - if (BOND_MODULO_SHORTCUT (slave_count)) - return (c & (slave_count - 1)); - else - return c % slave_count; } else if (ip_version == 0x6) { @@ -349,45 +381,300 @@ bond_load_balance_l34 (vlib_main_t * vm, vlib_node_runtime_t * node, uword), clib_mem_unaligned (&ip6->dst_address.as_uword[1], uword), a); - if (BOND_MODULO_SHORTCUT (slave_count)) - return (c & (slave_count - 1)); - else - return c % slave_count; + return c; } - return (bond_load_balance_l2 (vm, node, bif, b0, slave_count)); + return bond_lb_l2 (b0); } static_always_inline u32 -bond_load_balance_round_robin (vlib_main_t * vm, - vlib_node_runtime_t * node, - bond_if_t * bif, vlib_buffer_t * b0, - uword slave_count) +bond_lb_round_robin (bond_if_t * bif, vlib_buffer_t * b0, uword n_members) { bif->lb_rr_last_index++; - if (BOND_MODULO_SHORTCUT (slave_count)) - bif->lb_rr_last_index &= slave_count - 1; - else - bif->lb_rr_last_index %= slave_count; + if (bif->lb_rr_last_index >= n_members) + bif->lb_rr_last_index = 0; return bif->lb_rr_last_index; } -static_always_inline u32 -bond_load_balance_active_backup (vlib_main_t * vm, - vlib_node_runtime_t * node, - bond_if_t * bif, vlib_buffer_t * b0, - uword slave_count) +static_always_inline void +bond_tx_inline (vlib_main_t * vm, bond_if_t * bif, vlib_buffer_t ** b, + u32 * h, u32 n_left, uword n_members, u32 lb_alg) { - /* First interface is the active, the rest is backup */ - return 0; + while (n_left >= 4) + { + // Prefetch next iteration + if (n_left >= 8) + { + vlib_buffer_t **pb = b + 4; + + vlib_prefetch_buffer_header (pb[0], LOAD); + vlib_prefetch_buffer_header (pb[1], LOAD); + vlib_prefetch_buffer_header (pb[2], LOAD); + vlib_prefetch_buffer_header (pb[3], LOAD); + + CLIB_PREFETCH (pb[0]->data, CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (pb[1]->data, CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (pb[2]->data, CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (pb[3]->data, CLIB_CACHE_LINE_BYTES, LOAD); + } + + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]); + + if (lb_alg == BOND_LB_L2) + { + h[0] = bond_lb_l2 (b[0]); + h[1] = bond_lb_l2 (b[1]); + h[2] = bond_lb_l2 (b[2]); + h[3] = bond_lb_l2 (b[3]); + } + else if (lb_alg == BOND_LB_L34) + { + h[0] = bond_lb_l34 (b[0]); + h[1] = bond_lb_l34 (b[1]); + h[2] = bond_lb_l34 (b[2]); + h[3] = bond_lb_l34 (b[3]); + } + else if (lb_alg == BOND_LB_L23) + { + h[0] = bond_lb_l23 (b[0]); + h[1] = bond_lb_l23 (b[1]); + h[2] = bond_lb_l23 (b[2]); + h[3] = bond_lb_l23 (b[3]); + } + else if (lb_alg == BOND_LB_RR) + { + h[0] = bond_lb_round_robin (bif, b[0], n_members); + h[1] = bond_lb_round_robin (bif, b[1], n_members); + h[2] = bond_lb_round_robin (bif, b[2], n_members); + h[3] = bond_lb_round_robin (bif, b[3], n_members); + } + else if (lb_alg == BOND_LB_BC) + { + h[0] = bond_lb_broadcast (vm, bif, b[0], n_members); + h[1] = bond_lb_broadcast (vm, bif, b[1], n_members); + h[2] = bond_lb_broadcast (vm, bif, b[2], n_members); + h[3] = bond_lb_broadcast (vm, bif, b[3], n_members); + } + else + { + ASSERT (0); + } + + n_left -= 4; + b += 4; + h += 4; + } + + while (n_left > 0) + { + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); + + if (bif->lb == BOND_LB_L2) + h[0] = bond_lb_l2 (b[0]); + else if (bif->lb == BOND_LB_L34) + h[0] = bond_lb_l34 (b[0]); + else if (bif->lb == BOND_LB_L23) + h[0] = bond_lb_l23 (b[0]); + else if (bif->lb == BOND_LB_RR) + h[0] = bond_lb_round_robin (bif, b[0], n_members); + else if (bif->lb == BOND_LB_BC) + h[0] = bond_lb_broadcast (vm, bif, b[0], n_members); + else + { + ASSERT (0); + } + + n_left -= 1; + b += 1; + h += 1; + } } -static bond_load_balance_func_t bond_load_balance_table[] = { -#define _(v,f,s, p) { bond_load_balance_##p }, - foreach_bond_lb_algo -#undef _ -}; +static_always_inline void +bond_hash_to_port (u32 * h, u32 n_left, u32 n_members, + int use_modulo_shortcut) +{ + u32 mask = n_members - 1; + +#ifdef CLIB_HAVE_VEC256 + /* only lower 16 bits of hash due to single precision fp arithmetic */ + u32x8 mask8, sc8u, h8a, h8b; + f32x8 sc8f; + + if (use_modulo_shortcut) + { + mask8 = u32x8_splat (mask); + } + else + { + mask8 = u32x8_splat (0xffff); + sc8u = u32x8_splat (n_members); + sc8f = f32x8_from_u32x8 (sc8u); + } + + while (n_left > 16) + { + h8a = u32x8_load_unaligned (h) & mask8; + h8b = u32x8_load_unaligned (h + 8) & mask8; + + if (use_modulo_shortcut == 0) + { + h8a -= sc8u * u32x8_from_f32x8 (f32x8_from_u32x8 (h8a) / sc8f); + h8b -= sc8u * u32x8_from_f32x8 (f32x8_from_u32x8 (h8b) / sc8f); + } + + u32x8_store_unaligned (h8a, h); + u32x8_store_unaligned (h8b, h + 8); + n_left -= 16; + h += 16; + } +#endif + + while (n_left > 4) + { + if (use_modulo_shortcut) + { + h[0] &= mask; + h[1] &= mask; + h[2] &= mask; + h[3] &= mask; + } + else + { + h[0] %= n_members; + h[1] %= n_members; + h[2] %= n_members; + h[3] %= n_members; + } + n_left -= 4; + h += 4; + } + while (n_left) + { + if (use_modulo_shortcut) + h[0] &= mask; + else + h[0] %= n_members; + n_left -= 1; + h += 1; + } +} + +static_always_inline void +bond_update_sw_if_index (bond_per_thread_data_t * ptd, bond_if_t * bif, + u32 * bi, vlib_buffer_t ** b, u32 * data, u32 n_left, + int single_sw_if_index) +{ + u32 sw_if_index = data[0]; + u32 *h = data; + + while (n_left >= 4) + { + // Prefetch next iteration + if (n_left >= 8) + { + vlib_buffer_t **pb = b + 4; + vlib_prefetch_buffer_header (pb[0], LOAD); + vlib_prefetch_buffer_header (pb[1], LOAD); + vlib_prefetch_buffer_header (pb[2], LOAD); + vlib_prefetch_buffer_header (pb[3], LOAD); + } + + if (PREDICT_FALSE (single_sw_if_index)) + { + vnet_buffer (b[0])->sw_if_index[VLIB_TX] = sw_if_index; + vnet_buffer (b[1])->sw_if_index[VLIB_TX] = sw_if_index; + vnet_buffer (b[2])->sw_if_index[VLIB_TX] = sw_if_index; + vnet_buffer (b[3])->sw_if_index[VLIB_TX] = sw_if_index; + + bond_tx_add_to_queue (ptd, 0, bi[0]); + bond_tx_add_to_queue (ptd, 0, bi[1]); + bond_tx_add_to_queue (ptd, 0, bi[2]); + bond_tx_add_to_queue (ptd, 0, bi[3]); + } + else + { + u32 sw_if_index[4]; + + sw_if_index[0] = *vec_elt_at_index (bif->active_members, h[0]); + sw_if_index[1] = *vec_elt_at_index (bif->active_members, h[1]); + sw_if_index[2] = *vec_elt_at_index (bif->active_members, h[2]); + sw_if_index[3] = *vec_elt_at_index (bif->active_members, h[3]); + + vnet_buffer (b[0])->sw_if_index[VLIB_TX] = sw_if_index[0]; + vnet_buffer (b[1])->sw_if_index[VLIB_TX] = sw_if_index[1]; + vnet_buffer (b[2])->sw_if_index[VLIB_TX] = sw_if_index[2]; + vnet_buffer (b[3])->sw_if_index[VLIB_TX] = sw_if_index[3]; + + bond_tx_add_to_queue (ptd, h[0], bi[0]); + bond_tx_add_to_queue (ptd, h[1], bi[1]); + bond_tx_add_to_queue (ptd, h[2], bi[2]); + bond_tx_add_to_queue (ptd, h[3], bi[3]); + } + + bi += 4; + h += 4; + b += 4; + n_left -= 4; + } + while (n_left) + { + if (PREDICT_FALSE (single_sw_if_index)) + { + vnet_buffer (b[0])->sw_if_index[VLIB_TX] = sw_if_index; + bond_tx_add_to_queue (ptd, 0, bi[0]); + } + else + { + u32 sw_if_index0 = *vec_elt_at_index (bif->active_members, h[0]); + + vnet_buffer (b[0])->sw_if_index[VLIB_TX] = sw_if_index0; + bond_tx_add_to_queue (ptd, h[0], bi[0]); + } + + bi += 1; + h += 1; + b += 1; + n_left -= 1; + } +} + +static_always_inline void +bond_tx_trace (vlib_main_t * vm, vlib_node_runtime_t * node, bond_if_t * bif, + vlib_buffer_t ** b, u32 n_left, u32 * h) +{ + uword n_trace = vlib_get_trace_count (vm, node); + + while (n_trace > 0 && n_left > 0) + { + bond_packet_trace_t *t0; + ethernet_header_t *eth; + u32 next0 = 0; + + vlib_trace_buffer (vm, node, next0, b[0], 0 /* follow_chain */ ); + vlib_set_trace_count (vm, node, --n_trace); + t0 = vlib_add_trace (vm, node, b[0], sizeof (*t0)); + eth = vlib_buffer_get_current (b[0]); + t0->ethernet = *eth; + t0->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; + if (!h) + { + t0->bond_sw_if_index = *vec_elt_at_index (bif->active_members, 0); + } + else + { + t0->bond_sw_if_index = + *vec_elt_at_index (bif->active_members, h[0]); + h++; + } + b++; + n_left--; + } +} VNET_DEVICE_CLASS_TX_FN (bond_dev_class) (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -395,24 +682,21 @@ VNET_DEVICE_CLASS_TX_FN (bond_dev_class) (vlib_main_t * vm, { vnet_interface_output_runtime_t *rund = (void *) node->runtime_data; bond_main_t *bm = &bond_main; + u16 thread_index = vm->thread_index; bond_if_t *bif = pool_elt_at_index (bm->interfaces, rund->dev_instance); - vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; + uword n_members; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE]; u32 *from = vlib_frame_vector_args (frame); - ethernet_header_t *eth; - u32 port, n_left; - u32 sw_if_index, sw_if_index1, sw_if_index2, sw_if_index3; - bond_packet_trace_t *t0; - uword n_trace = vlib_get_trace_count (vm, node); - u16 thread_index = vm->thread_index; + u32 n_left = frame->n_vectors; + u32 hashes[VLIB_FRAME_SIZE], *h; vnet_main_t *vnm = vnet_get_main (); - u32 *to_next; - u32 sif_if_index, sif_if_index1, sif_if_index2, sif_if_index3; - vlib_frame_t *f; - uword slave_count; + bond_per_thread_data_t *ptd = vec_elt_at_index (bm->per_thread_data, + thread_index); + u32 p, sw_if_index; if (PREDICT_FALSE (bif->admin_up == 0)) { - vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors); + vlib_buffer_free (vm, vlib_frame_vector_args (frame), frame->n_vectors); vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters + VNET_INTERFACE_COUNTER_DROP, thread_index, bif->sw_if_index, @@ -422,241 +706,89 @@ VNET_DEVICE_CLASS_TX_FN (bond_dev_class) (vlib_main_t * vm, return frame->n_vectors; } - n_left = frame->n_vectors; - vlib_get_buffers (vm, from, bufs, n_left); - - slave_count = vec_len (bif->active_slaves); - if (PREDICT_FALSE (slave_count == 0)) + n_members = vec_len (bif->active_members); + if (PREDICT_FALSE (n_members == 0)) { - vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors); + vlib_buffer_free (vm, vlib_frame_vector_args (frame), frame->n_vectors); vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters + VNET_INTERFACE_COUNTER_DROP, thread_index, bif->sw_if_index, frame->n_vectors); - vlib_error_count (vm, node->node_index, BOND_TX_ERROR_NO_SLAVE, + vlib_error_count (vm, node->node_index, BOND_TX_ERROR_NO_MEMBER, frame->n_vectors); return frame->n_vectors; } - vec_validate_aligned (bif->per_thread_info[thread_index].frame, slave_count, - CLIB_CACHE_LINE_BYTES); + vlib_get_buffers (vm, from, bufs, n_left); - b = bufs; - while (n_left >= 4) + /* active-backup mode, ship everything to first sw if index */ + if ((bif->lb == BOND_LB_AB) || PREDICT_FALSE (n_members == 1)) { - u32 next0 = 0, next1 = 0, next2 = 0, next3 = 0; - u32 port0 = 0, port1 = 0, port2 = 0, port3 = 0; + sw_if_index = *vec_elt_at_index (bif->active_members, 0); - // Prefetch next iteration - if (n_left >= 8) - { - vlib_buffer_t **pb = b + 4; + bond_tx_trace (vm, node, bif, bufs, frame->n_vectors, 0); + bond_update_sw_if_index (ptd, bif, from, bufs, &sw_if_index, n_left, + /* single_sw_if_index */ 1); + goto done; + } - vlib_prefetch_buffer_header (pb[0], LOAD); - vlib_prefetch_buffer_header (pb[1], LOAD); - vlib_prefetch_buffer_header (pb[2], LOAD); - vlib_prefetch_buffer_header (pb[3], LOAD); + if (bif->lb == BOND_LB_BC) + { + sw_if_index = *vec_elt_at_index (bif->active_members, 0); - CLIB_PREFETCH (pb[0]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (pb[1]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (pb[2]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (pb[3]->data, CLIB_CACHE_LINE_BYTES, LOAD); - } + bond_tx_inline (vm, bif, bufs, hashes, n_left, n_members, BOND_LB_BC); + bond_tx_trace (vm, node, bif, bufs, frame->n_vectors, 0); + bond_update_sw_if_index (ptd, bif, from, bufs, &sw_if_index, n_left, + /* single_sw_if_index */ 1); + goto done; + } - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]); + /* if have at least one member on local numa node, only members on local numa + node will transmit pkts when bif->local_numa_only is enabled */ + if (bif->n_numa_members >= 1) + n_members = bif->n_numa_members; + + if (bif->lb == BOND_LB_L2) + bond_tx_inline (vm, bif, bufs, hashes, n_left, n_members, BOND_LB_L2); + else if (bif->lb == BOND_LB_L34) + bond_tx_inline (vm, bif, bufs, hashes, n_left, n_members, BOND_LB_L34); + else if (bif->lb == BOND_LB_L23) + bond_tx_inline (vm, bif, bufs, hashes, n_left, n_members, BOND_LB_L23); + else if (bif->lb == BOND_LB_RR) + bond_tx_inline (vm, bif, bufs, hashes, n_left, n_members, BOND_LB_RR); + else + ASSERT (0); - sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; - sw_if_index1 = vnet_buffer (b[1])->sw_if_index[VLIB_TX]; - sw_if_index2 = vnet_buffer (b[2])->sw_if_index[VLIB_TX]; - sw_if_index3 = vnet_buffer (b[3])->sw_if_index[VLIB_TX]; + /* calculate port out of hash */ + h = hashes; + if (BOND_MODULO_SHORTCUT (n_members)) + bond_hash_to_port (h, frame->n_vectors, n_members, 1); + else + bond_hash_to_port (h, frame->n_vectors, n_members, 0); - if (PREDICT_TRUE (slave_count > 1)) - { - port0 = - (bond_load_balance_table[bif->lb]).load_balance (vm, node, - bif, b[0], - slave_count); - port1 = - (bond_load_balance_table[bif->lb]).load_balance (vm, node, - bif, b[1], - slave_count); - port2 = - (bond_load_balance_table[bif->lb]).load_balance (vm, node, - bif, b[2], - slave_count); - port3 = - (bond_load_balance_table[bif->lb]).load_balance (vm, node, - bif, b[3], - slave_count); - } + bond_tx_trace (vm, node, bif, bufs, frame->n_vectors, h); - sif_if_index = *vec_elt_at_index (bif->active_slaves, port0); - sif_if_index1 = *vec_elt_at_index (bif->active_slaves, port1); - sif_if_index2 = *vec_elt_at_index (bif->active_slaves, port2); - sif_if_index3 = *vec_elt_at_index (bif->active_slaves, port3); - - vnet_buffer (b[0])->sw_if_index[VLIB_TX] = sif_if_index; - vnet_buffer (b[1])->sw_if_index[VLIB_TX] = sif_if_index1; - vnet_buffer (b[2])->sw_if_index[VLIB_TX] = sif_if_index2; - vnet_buffer (b[3])->sw_if_index[VLIB_TX] = sif_if_index3; - - if (PREDICT_FALSE ((bif->per_thread_info[thread_index].frame[port0] - == 0))) - bif->per_thread_info[thread_index].frame[port0] = - vnet_get_frame_to_sw_interface (vnm, sif_if_index); - - if (PREDICT_FALSE ((bif->per_thread_info[thread_index].frame[port1] - == 0))) - bif->per_thread_info[thread_index].frame[port1] = - vnet_get_frame_to_sw_interface (vnm, sif_if_index1); - - if (PREDICT_FALSE ((bif->per_thread_info[thread_index].frame[port2] - == 0))) - bif->per_thread_info[thread_index].frame[port2] = - vnet_get_frame_to_sw_interface (vnm, sif_if_index2); - - if (PREDICT_FALSE ((bif->per_thread_info[thread_index].frame[port3] - == 0))) - bif->per_thread_info[thread_index].frame[port3] = - vnet_get_frame_to_sw_interface (vnm, sif_if_index3); - - f = bif->per_thread_info[thread_index].frame[port0]; - to_next = vlib_frame_vector_args (f); - to_next += f->n_vectors; - to_next[0] = vlib_get_buffer_index (vm, b[0]); - f->n_vectors++; - - f = bif->per_thread_info[thread_index].frame[port1]; - to_next = vlib_frame_vector_args (f); - to_next += f->n_vectors; - to_next[0] = vlib_get_buffer_index (vm, b[1]); - f->n_vectors++; - - f = bif->per_thread_info[thread_index].frame[port2]; - to_next = vlib_frame_vector_args (f); - to_next += f->n_vectors; - to_next[0] = vlib_get_buffer_index (vm, b[2]); - f->n_vectors++; - - f = bif->per_thread_info[thread_index].frame[port3]; - to_next = vlib_frame_vector_args (f); - to_next += f->n_vectors; - to_next[0] = vlib_get_buffer_index (vm, b[3]); - f->n_vectors++; - - if (PREDICT_FALSE (n_trace > 0)) - { - vlib_trace_buffer (vm, node, next0, b[0], 0 /* follow_chain */ ); - vlib_set_trace_count (vm, node, --n_trace); - t0 = vlib_add_trace (vm, node, b[0], sizeof (*t0)); - eth = (ethernet_header_t *) vlib_buffer_get_current (b[0]); - t0->ethernet = *eth; - t0->sw_if_index = sw_if_index; - t0->bond_sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; - - if (PREDICT_TRUE (n_trace > 0)) - { - vlib_trace_buffer (vm, node, next1, b[1], - 0 /* follow_chain */ ); - vlib_set_trace_count (vm, node, --n_trace); - t0 = vlib_add_trace (vm, node, b[1], sizeof (*t0)); - eth = (ethernet_header_t *) vlib_buffer_get_current (b[1]); - t0->ethernet = *eth; - t0->sw_if_index = sw_if_index1; - t0->bond_sw_if_index = vnet_buffer (b[1])->sw_if_index[VLIB_TX]; - - if (PREDICT_TRUE (n_trace > 0)) - { - vlib_trace_buffer (vm, node, next2, b[2], - 0 /* follow_chain */ ); - vlib_set_trace_count (vm, node, --n_trace); - t0 = vlib_add_trace (vm, node, b[2], sizeof (*t0)); - eth = (ethernet_header_t *) vlib_buffer_get_current (b[2]); - t0->ethernet = *eth; - t0->sw_if_index = sw_if_index2; - t0->bond_sw_if_index = - vnet_buffer (b[2])->sw_if_index[VLIB_TX]; - - if (PREDICT_TRUE (n_trace > 0)) - { - vlib_trace_buffer (vm, node, next3, b[3], - 0 /* follow_chain */ ); - vlib_set_trace_count (vm, node, --n_trace); - t0 = vlib_add_trace (vm, node, b[3], sizeof (*t0)); - eth = - (ethernet_header_t *) vlib_buffer_get_current (b[3]); - t0->ethernet = *eth; - t0->sw_if_index = sw_if_index3; - t0->bond_sw_if_index = - vnet_buffer (b[3])->sw_if_index[VLIB_TX]; - } - } - } - } - from += 4; - n_left -= 4; - b += 4; - } + bond_update_sw_if_index (ptd, bif, from, bufs, hashes, frame->n_vectors, + /* single_sw_if_index */ 0); - while (n_left > 0) +done: + for (p = 0; p < n_members; p++) { - u32 next0 = 0; - u32 port0 = 0; - - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); + vlib_frame_t *f; + u32 *to_next; - sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; - - if (PREDICT_TRUE (slave_count > 1)) - port0 = - (bond_load_balance_table[bif->lb]).load_balance (vm, node, bif, - b[0], slave_count); - sif_if_index = *vec_elt_at_index (bif->active_slaves, port0); - vnet_buffer (b[0])->sw_if_index[VLIB_TX] = sif_if_index; - if (PREDICT_FALSE - ((bif->per_thread_info[thread_index].frame[port0] == 0))) - bif->per_thread_info[thread_index].frame[port0] = - vnet_get_frame_to_sw_interface (vnm, sif_if_index); - f = bif->per_thread_info[thread_index].frame[port0]; - to_next = vlib_frame_vector_args (f); - to_next += f->n_vectors; - to_next[0] = vlib_get_buffer_index (vm, b[0]); - f->n_vectors++; - - if (PREDICT_FALSE (n_trace > 0)) + sw_if_index = *vec_elt_at_index (bif->active_members, p); + if (PREDICT_TRUE (ptd->per_port_queue[p].n_buffers)) { - vlib_trace_buffer (vm, node, next0, b[0], 0 /* follow_chain */ ); - vlib_set_trace_count (vm, node, --n_trace); - t0 = vlib_add_trace (vm, node, b[0], sizeof (*t0)); - eth = (ethernet_header_t *) vlib_buffer_get_current (b[0]); - t0->ethernet = *eth; - t0->sw_if_index = sw_if_index; - t0->bond_sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; + f = vnet_get_frame_to_sw_interface (vnm, sw_if_index); + f->n_vectors = ptd->per_port_queue[p].n_buffers; + to_next = vlib_frame_vector_args (f); + clib_memcpy_fast (to_next, ptd->per_port_queue[p].buffers, + f->n_vectors * sizeof (u32)); + vnet_put_frame_to_sw_interface (vnm, sw_if_index, f); + ptd->per_port_queue[p].n_buffers = 0; } - - from += 1; - n_left -= 1; - b += 1; } - - for (port = 0; port < slave_count; port++) - { - f = bif->per_thread_info[thread_index].frame[port]; - if (f == 0) - continue; - - sw_if_index = *vec_elt_at_index (bif->active_slaves, port); - vnet_put_frame_to_sw_interface (vnm, sw_if_index, f); - bif->per_thread_info[thread_index].frame[port] = 0; - } - - vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters - + VNET_INTERFACE_COUNTER_TX, thread_index, - bif->sw_if_index, frame->n_vectors); - return frame->n_vectors; } @@ -666,8 +798,7 @@ bond_active_interface_switch_cb (vnet_main_t * vnm, u32 sw_if_index, { bond_main_t *bm = &bond_main; - send_ip4_garp (bm->vlib_main, sw_if_index); - send_ip6_na (bm->vlib_main, sw_if_index); + ip_neighbor_advertise (bm->vlib_main, IP46_TYPE_BOTH, NULL, sw_if_index); return (WALK_CONTINUE); } @@ -689,9 +820,10 @@ bond_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) for (i = 0; i < vec_len (event_data); i++) { hw_if_index = event_data[i]; - /* walk hw interface to process all subinterfaces */ - vnet_hw_interface_walk_sw (vnm, hw_if_index, - bond_active_interface_switch_cb, 0); + if (vnet_get_hw_interface_or_null (vnm, hw_if_index)) + /* walk hw interface to process all subinterfaces */ + vnet_hw_interface_walk_sw (vnm, hw_if_index, + bond_active_interface_switch_cb, 0); } vec_reset_length (event_data); } @@ -701,6 +833,7 @@ bond_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) /* *INDENT-OFF* */ VLIB_REGISTER_NODE (bond_process_node) = { .function = bond_process, + .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED, .type = VLIB_NODE_TYPE_PROCESS, .name = "bond-process", }; @@ -716,10 +849,30 @@ VNET_DEVICE_CLASS (bond_dev_class) = { .admin_up_down_function = bond_interface_admin_up_down, .subif_add_del_function = bond_subif_add_del_function, .format_tx_trace = format_bond_tx_trace, + .mac_addr_add_del_function = bond_add_del_mac_address, }; /* *INDENT-ON* */ +static clib_error_t * +bond_member_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add) +{ + bond_main_t *bm = &bond_main; + member_if_t *mif; + bond_detach_member_args_t args = { 0 }; + + if (is_add) + return 0; + mif = bond_get_member_by_sw_if_index (sw_if_index); + if (!mif) + return 0; + args.member = sw_if_index; + bond_detach_member (bm->vlib_main, &args); + return args.error; +} + +VNET_SW_INTERFACE_ADD_DEL_FUNCTION (bond_member_interface_add_del); + /* * fd.io coding-style-patch-verification: ON *