X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fip%2Fip6_forward.c;h=b6eae6e381175d405e68976d9ef36a0afc693543;hb=b538dd868;hp=232f72833fd619c3f9dbc32194dbee211d532bf4;hpb=044183faacc5eb8a055658f9deebefb56b254adc;p=vpp.git diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index 232f72833fd..b6eae6e3811 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -39,281 +39,25 @@ #include #include +#include +#include #include /* for ethernet_header_t */ #include /* for srp_hw_interface_class */ #include -#include +#include /* for FIB uRPF check */ #include -#include +#include +#include #include +#ifndef CLIB_MARCH_VARIANT #include +#endif +#include +#include -/** - * @file - * @brief IPv6 Forwarding. - * - * This file contains the source code for IPv6 forwarding. - */ - -void -ip6_forward_next_trace (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame, - vlib_rx_or_tx_t which_adj_index); - -always_inline uword -ip6_lookup_inline (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - ip6_main_t *im = &ip6_main; - vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters; - u32 n_left_from, n_left_to_next, *from, *to_next; - ip_lookup_next_t next; - u32 cpu_index = os_get_cpu_number (); - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next = node->cached_next_index; - - while (n_left_from > 0) - { - vlib_get_next_frame (vm, node, next, to_next, n_left_to_next); - - while (n_left_from >= 4 && n_left_to_next >= 2) - { - vlib_buffer_t *p0, *p1; - u32 pi0, pi1, lbi0, lbi1, wrong_next; - ip_lookup_next_t next0, next1; - ip6_header_t *ip0, *ip1; - ip6_address_t *dst_addr0, *dst_addr1; - u32 fib_index0, fib_index1; - u32 flow_hash_config0, flow_hash_config1; - const dpo_id_t *dpo0, *dpo1; - const load_balance_t *lb0, *lb1; - - /* Prefetch next iteration. */ - { - vlib_buffer_t *p2, *p3; - - p2 = vlib_get_buffer (vm, from[2]); - p3 = vlib_get_buffer (vm, from[3]); - - vlib_prefetch_buffer_header (p2, LOAD); - vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD); - CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD); - } - - pi0 = to_next[0] = from[0]; - pi1 = to_next[1] = from[1]; - - p0 = vlib_get_buffer (vm, pi0); - p1 = vlib_get_buffer (vm, pi1); - - ip0 = vlib_buffer_get_current (p0); - ip1 = vlib_buffer_get_current (p1); - - dst_addr0 = &ip0->dst_address; - dst_addr1 = &ip1->dst_address; - - fib_index0 = - vec_elt (im->fib_index_by_sw_if_index, - vnet_buffer (p0)->sw_if_index[VLIB_RX]); - fib_index1 = - vec_elt (im->fib_index_by_sw_if_index, - vnet_buffer (p1)->sw_if_index[VLIB_RX]); - - fib_index0 = (vnet_buffer (p0)->sw_if_index[VLIB_TX] == (u32) ~ 0) ? - fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX]; - fib_index1 = (vnet_buffer (p1)->sw_if_index[VLIB_TX] == (u32) ~ 0) ? - fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX]; - - lbi0 = ip6_fib_table_fwding_lookup (im, fib_index0, dst_addr0); - lbi1 = ip6_fib_table_fwding_lookup (im, fib_index1, dst_addr1); - - lb0 = load_balance_get (lbi0); - lb1 = load_balance_get (lbi1); - - vnet_buffer (p0)->ip.flow_hash = vnet_buffer (p1)->ip.flow_hash = 0; - - if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) - { - flow_hash_config0 = lb0->lb_hash_config; - vnet_buffer (p0)->ip.flow_hash = - ip6_compute_flow_hash (ip0, flow_hash_config0); - } - if (PREDICT_FALSE (lb1->lb_n_buckets > 1)) - { - flow_hash_config1 = lb1->lb_hash_config; - vnet_buffer (p1)->ip.flow_hash = - ip6_compute_flow_hash (ip1, flow_hash_config1); - } - - ASSERT (lb0->lb_n_buckets > 0); - ASSERT (lb1->lb_n_buckets > 0); - ASSERT (is_pow2 (lb0->lb_n_buckets)); - ASSERT (is_pow2 (lb1->lb_n_buckets)); - dpo0 = load_balance_get_bucket_i (lb0, - (vnet_buffer (p0)->ip.flow_hash & - lb0->lb_n_buckets_minus_1)); - dpo1 = load_balance_get_bucket_i (lb1, - (vnet_buffer (p1)->ip.flow_hash & - lb1->lb_n_buckets_minus_1)); - - next0 = dpo0->dpoi_next_node; - next1 = dpo1->dpoi_next_node; - - /* Only process the HBH Option Header if explicitly configured to do so */ - if (PREDICT_FALSE - (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) - { - next0 = (dpo_is_adj (dpo0) && im->hbh_enabled) ? - (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0; - } - if (PREDICT_FALSE - (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) - { - next1 = (dpo_is_adj (dpo1) && im->hbh_enabled) ? - (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next1; - } - vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; - - vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); - vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); - - from += 2; - to_next += 2; - n_left_to_next -= 2; - n_left_from -= 2; - - wrong_next = (next0 != next) + 2 * (next1 != next); - if (PREDICT_FALSE (wrong_next != 0)) - { - switch (wrong_next) - { - case 1: - /* A B A */ - to_next[-2] = pi1; - to_next -= 1; - n_left_to_next += 1; - vlib_set_next_frame_buffer (vm, node, next0, pi0); - break; - - case 2: - /* A A B */ - to_next -= 1; - n_left_to_next += 1; - vlib_set_next_frame_buffer (vm, node, next1, pi1); - break; - - case 3: - /* A B C */ - to_next -= 2; - n_left_to_next += 2; - vlib_set_next_frame_buffer (vm, node, next0, pi0); - vlib_set_next_frame_buffer (vm, node, next1, pi1); - if (next0 == next1) - { - /* A B B */ - vlib_put_next_frame (vm, node, next, n_left_to_next); - next = next1; - vlib_get_next_frame (vm, node, next, to_next, - n_left_to_next); - } - } - } - } - - while (n_left_from > 0 && n_left_to_next > 0) - { - vlib_buffer_t *p0; - ip6_header_t *ip0; - u32 pi0, lbi0; - ip_lookup_next_t next0; - load_balance_t *lb0; - ip6_address_t *dst_addr0; - u32 fib_index0, flow_hash_config0; - const dpo_id_t *dpo0; - - pi0 = from[0]; - to_next[0] = pi0; - - p0 = vlib_get_buffer (vm, pi0); - - ip0 = vlib_buffer_get_current (p0); - - dst_addr0 = &ip0->dst_address; - - fib_index0 = - vec_elt (im->fib_index_by_sw_if_index, - vnet_buffer (p0)->sw_if_index[VLIB_RX]); - fib_index0 = - (vnet_buffer (p0)->sw_if_index[VLIB_TX] == - (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX]; - - flow_hash_config0 = ip6_fib_get (fib_index0)->flow_hash_config; - - lbi0 = ip6_fib_table_fwding_lookup (im, fib_index0, dst_addr0); - - lb0 = load_balance_get (lbi0); - - vnet_buffer (p0)->ip.flow_hash = 0; - - if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) - { - flow_hash_config0 = lb0->lb_hash_config; - vnet_buffer (p0)->ip.flow_hash = - ip6_compute_flow_hash (ip0, flow_hash_config0); - } - - ASSERT (lb0->lb_n_buckets > 0); - ASSERT (is_pow2 (lb0->lb_n_buckets)); - dpo0 = load_balance_get_bucket_i (lb0, - (vnet_buffer (p0)->ip.flow_hash & - lb0->lb_n_buckets_minus_1)); - next0 = dpo0->dpoi_next_node; - - /* Only process the HBH Option Header if explicitly configured to do so */ - if (PREDICT_FALSE - (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) - { - next0 = (dpo_is_adj (dpo0) && im->hbh_enabled) ? - (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0; - } - vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - - vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); - - from += 1; - to_next += 1; - n_left_to_next -= 1; - n_left_from -= 1; - - if (PREDICT_FALSE (next0 != next)) - { - n_left_to_next += 1; - vlib_put_next_frame (vm, node, next, n_left_to_next); - next = next0; - vlib_get_next_frame (vm, node, next, to_next, n_left_to_next); - to_next[0] = pi0; - to_next += 1; - n_left_to_next -= 1; - } - } - - vlib_put_next_frame (vm, node, next, n_left_to_next); - } - - if (node->flags & VLIB_NODE_FLAG_TRACE) - ip6_forward_next_trace (vm, node, frame, VLIB_TX); - - return frame->n_vectors; -} +/* Flag used by IOAM code. Classifier sets it pop-hop-by-hop checks it */ +#define OI_DECAP 0x80000000 static void ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index, @@ -328,16 +72,20 @@ ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index, .fp_addr.ip6 = *address, }; - a->neighbor_probe_adj_index = ~0; if (a->address_length < 128) { - fib_node_index_t fei; - - fei = fib_table_entry_update_one_path (fib_index, &pfx, FIB_SOURCE_INTERFACE, (FIB_ENTRY_FLAG_CONNECTED | FIB_ENTRY_FLAG_ATTACHED), FIB_PROTOCOL_IP6, NULL, /* No next-hop address */ - sw_if_index, ~0, // invalid FIB index - 1, NULL, // no label stack - FIB_ROUTE_PATH_FLAG_NONE); - a->neighbor_probe_adj_index = fib_entry_get_adj (fei); + fib_table_entry_update_one_path (fib_index, + &pfx, + FIB_SOURCE_INTERFACE, + (FIB_ENTRY_FLAG_CONNECTED | + FIB_ENTRY_FLAG_ATTACHED), + DPO_PROTO_IP6, + /* No next-hop address */ + NULL, sw_if_index, + /* invalid FIB index */ + ~0, 1, + /* no label stack */ + NULL, FIB_ROUTE_PATH_FLAG_NONE); } pfx.fp_len = 128; @@ -362,7 +110,13 @@ ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index, } } - fib_table_entry_update_one_path (fib_index, &pfx, FIB_SOURCE_INTERFACE, (FIB_ENTRY_FLAG_CONNECTED | FIB_ENTRY_FLAG_LOCAL), FIB_PROTOCOL_IP6, &pfx.fp_addr, sw_if_index, ~0, // invalid FIB index + fib_table_entry_update_one_path (fib_index, &pfx, + FIB_SOURCE_INTERFACE, + (FIB_ENTRY_FLAG_CONNECTED | + FIB_ENTRY_FLAG_LOCAL), + DPO_PROTO_IP6, + &pfx.fp_addr, + sw_if_index, ~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE); } @@ -387,6 +141,7 @@ ip6_del_interface_routes (ip6_main_t * im, fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE); } +#ifndef CLIB_MARCH_VARIANT void ip6_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable) { @@ -411,19 +166,16 @@ ip6_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable) return; } - vnet_feature_enable_disable ("ip6-unicast", "ip6-lookup", sw_if_index, - is_enable, 0, 0); - - vnet_feature_enable_disable ("ip6-multicast", "ip6-lookup", sw_if_index, - is_enable, 0, 0); + vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index, + !is_enable, 0, 0); + vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled", + sw_if_index, !is_enable, 0, 0); } /* get first interface address */ ip6_address_t * -ip6_interface_first_address (ip6_main_t * im, - u32 sw_if_index, - ip_interface_address_t ** result_ia) +ip6_interface_first_address (ip6_main_t * im, u32 sw_if_index) { ip_lookup_main_t *lm = &im->lookup_main; ip_interface_address_t *ia = 0; @@ -438,8 +190,6 @@ ip6_interface_first_address (ip6_main_t * im, break; })); /* *INDENT-ON* */ - if (result_ia) - *result_ia = result ? ia : 0; return result; } @@ -455,12 +205,96 @@ ip6_add_del_interface_address (vlib_main_t * vm, clib_error_t *error; u32 if_address_index; ip6_address_fib_t ip6_af, *addr_fib = 0; + ip6_address_t ll_addr; + + /* local0 interface doesn't support IP addressing */ + if (sw_if_index == 0) + { + return + clib_error_create ("local0 interface doesn't support IP addressing"); + } + + if (ip6_address_is_link_local_unicast (address)) + { + if (address_length != 128) + { + vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH; + return + clib_error_create + ("prefix length of link-local address must be 128"); + } + if (!is_del) + { + return ip6_neighbor_set_link_local_address (vm, sw_if_index, + address); + } + else + { + ll_addr = ip6_neighbor_get_link_local_address (sw_if_index); + if (ip6_address_is_equal (&ll_addr, address)) + { + vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_DELETABLE; + return clib_error_create ("address not deletable"); + } + else + { + vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE; + return clib_error_create ("address not found"); + } + } + } vec_validate (im->fib_index_by_sw_if_index, sw_if_index); + vec_validate (im->mfib_index_by_sw_if_index, sw_if_index); + ip6_addr_fib_init (&ip6_af, address, vec_elt (im->fib_index_by_sw_if_index, sw_if_index)); vec_add1 (addr_fib, ip6_af); + /* *INDENT-OFF* */ + if (!is_del) + { + /* When adding an address check that it does not conflict + with an existing address on any interface in this table. */ + ip_interface_address_t *ia; + vnet_sw_interface_t *sif; + + pool_foreach(sif, vnm->interface_main.sw_interfaces, + ({ + if (im->fib_index_by_sw_if_index[sw_if_index] == + im->fib_index_by_sw_if_index[sif->sw_if_index]) + { + foreach_ip_interface_address + (&im->lookup_main, ia, sif->sw_if_index, + 0 /* honor unnumbered */ , + ({ + ip6_address_t * x = + ip_interface_address_get_address + (&im->lookup_main, ia); + if (ip6_destination_matches_route + (im, address, x, ia->address_length) || + ip6_destination_matches_route (im, + x, + address, + address_length)) + { + vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS; + return + clib_error_create + ("failed to add %U which conflicts with %U for interface %U", + format_ip6_address_and_length, address, + address_length, + format_ip6_address_and_length, x, + ia->address_length, + format_vnet_sw_if_index_name, vnm, + sif->sw_if_index); + } + })); + } + })); + } + /* *INDENT-ON* */ + { uword elts_before = pool_elts (lm->if_address_pool); @@ -496,7 +330,9 @@ done: return error; } -clib_error_t * +#endif + +static clib_error_t * ip6_sw_interface_admin_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags) { ip6_main_t *im = &ip6_main; @@ -541,6 +377,7 @@ VNET_FEATURE_ARC_INIT (ip6_unicast, static) = { .arc_name = "ip6-unicast", .start_nodes = VNET_FEATURES ("ip6-input"), + .last_in_arc = "ip6-lookup", .arc_index_ptr = &ip6_main.lookup_main.ucast_feature_arc_index, }; @@ -562,13 +399,13 @@ VNET_FEATURE_INIT (ip6_policer_classify, static) = { .arc_name = "ip6-unicast", .node_name = "ip6-policer-classify", - .runs_before = VNET_FEATURES ("ipsec-input-ip6"), + .runs_before = VNET_FEATURES ("ipsec6-input-feature"), }; VNET_FEATURE_INIT (ip6_ipsec, static) = { .arc_name = "ip6-unicast", - .node_name = "ipsec-input-ip6", + .node_name = "ipsec6-input-feature", .runs_before = VNET_FEATURES ("l2tp-decap"), }; @@ -583,20 +420,27 @@ VNET_FEATURE_INIT (ip6_vpath, static) = { .arc_name = "ip6-unicast", .node_name = "vpath-input-ip6", + .runs_before = VNET_FEATURES ("ip6-vxlan-bypass"), +}; + +VNET_FEATURE_INIT (ip6_vxlan_bypass, static) = +{ + .arc_name = "ip6-unicast", + .node_name = "ip6-vxlan-bypass", .runs_before = VNET_FEATURES ("ip6-lookup"), }; -VNET_FEATURE_INIT (ip6_lookup, static) = +VNET_FEATURE_INIT (ip6_not_enabled, static) = { .arc_name = "ip6-unicast", - .node_name = "ip6-lookup", - .runs_before = VNET_FEATURES ("ip6-drop"), + .node_name = "ip6-not-enabled", + .runs_before = VNET_FEATURES ("ip6-lookup"), }; -VNET_FEATURE_INIT (ip6_drop, static) = +VNET_FEATURE_INIT (ip6_lookup, static) = { .arc_name = "ip6-unicast", - .node_name = "ip6-drop", + .node_name = "ip6-lookup", .runs_before = 0, /*last feature*/ }; @@ -605,24 +449,25 @@ VNET_FEATURE_ARC_INIT (ip6_multicast, static) = { .arc_name = "ip6-multicast", .start_nodes = VNET_FEATURES ("ip6-input"), + .last_in_arc = "ip6-mfib-forward-lookup", .arc_index_ptr = &ip6_main.lookup_main.mcast_feature_arc_index, }; VNET_FEATURE_INIT (ip6_vpath_mc, static) = { .arc_name = "ip6-multicast", .node_name = "vpath-input-ip6", - .runs_before = VNET_FEATURES ("ip6-lookup"), + .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"), }; -VNET_FEATURE_INIT (ip6_mc_lookup, static) = { +VNET_FEATURE_INIT (ip6_not_enabled_mc, static) = { .arc_name = "ip6-multicast", - .node_name = "ip6-lookup", - .runs_before = VNET_FEATURES ("ip6-drop"), + .node_name = "ip6-not-enabled", + .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"), }; -VNET_FEATURE_INIT (ip6_drop_mc, static) = { +VNET_FEATURE_INIT (ip6_mc_lookup, static) = { .arc_name = "ip6-multicast", - .node_name = "ip6-drop", + .node_name = "ip6-mfib-forward-lookup", .runs_before = 0, /* last feature */ }; @@ -630,13 +475,20 @@ VNET_FEATURE_INIT (ip6_drop_mc, static) = { VNET_FEATURE_ARC_INIT (ip6_output, static) = { .arc_name = "ip6-output", - .start_nodes = VNET_FEATURES ("ip6-rewrite", "ip6-midchain"), + .start_nodes = VNET_FEATURES ("ip6-rewrite", "ip6-midchain", "ip6-dvr-dpo"), + .last_in_arc = "interface-output", .arc_index_ptr = &ip6_main.lookup_main.output_feature_arc_index, }; +VNET_FEATURE_INIT (ip6_outacl, static) = { + .arc_name = "ip6-output", + .node_name = "ip6-outacl", + .runs_before = VNET_FEATURES ("ipsec6-output-feature"), +}; + VNET_FEATURE_INIT (ip6_ipsec_output, static) = { .arc_name = "ip6-output", - .node_name = "ipsec-output-ip6", + .node_name = "ipsec6-output-feature", .runs_before = VNET_FEATURES ("interface-output"), }; @@ -647,26 +499,49 @@ VNET_FEATURE_INIT (ip6_interface_output, static) = { }; /* *INDENT-ON* */ -clib_error_t * +static clib_error_t * ip6_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add) { - vnet_feature_enable_disable ("ip6-unicast", "ip6-drop", sw_if_index, - is_add, 0, 0); + ip6_main_t *im = &ip6_main; - vnet_feature_enable_disable ("ip6-multicast", "ip6-drop", sw_if_index, - is_add, 0, 0); + vec_validate (im->fib_index_by_sw_if_index, sw_if_index); + vec_validate (im->mfib_index_by_sw_if_index, sw_if_index); + + if (!is_add) + { + /* Ensure that IPv6 is disabled */ + ip6_main_t *im6 = &ip6_main; + ip_lookup_main_t *lm6 = &im6->lookup_main; + ip_interface_address_t *ia = 0; + ip6_address_t *address; + vlib_main_t *vm = vlib_get_main (); + + ip6_neighbor_sw_interface_add_del (vnm, sw_if_index, 0 /* is_add */ ); + vnet_sw_interface_update_unnumbered (sw_if_index, ~0, 0); + /* *INDENT-OFF* */ + foreach_ip_interface_address (lm6, ia, sw_if_index, 0, + ({ + address = ip_interface_address_get_address (lm6, ia); + ip6_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1); + })); + /* *INDENT-ON* */ + ip6_mfib_interface_enable_disable (sw_if_index, 0); + } - vnet_feature_enable_disable ("ip6-output", "interface-output", sw_if_index, + vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index, is_add, 0, 0); + vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled", + sw_if_index, is_add, 0, 0); + return /* no error */ 0; } VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del); -static uword -ip6_lookup (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) +VLIB_NODE_FN (ip6_lookup_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { return ip6_lookup_inline (vm, node, frame); } @@ -676,7 +551,6 @@ static u8 *format_ip6_lookup_trace (u8 * s, va_list * args); /* *INDENT-OFF* */ VLIB_REGISTER_NODE (ip6_lookup_node) = { - .function = ip6_lookup, .name = "ip6-lookup", .vector_size = sizeof (u32), .format_trace = format_ip6_lookup_trace, @@ -685,208 +559,178 @@ VLIB_REGISTER_NODE (ip6_lookup_node) = }; /* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (ip6_lookup_node, ip6_lookup); - -always_inline uword -ip6_load_balance (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) +VLIB_NODE_FN (ip6_load_balance_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters; - u32 n_left_from, n_left_to_next, *from, *to_next; - ip_lookup_next_t next; - u32 cpu_index = os_get_cpu_number (); + u32 n_left, *from; + u32 thread_index = vm->thread_index; ip6_main_t *im = &ip6_main; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; + u16 nexts[VLIB_FRAME_SIZE], *next; from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next = node->cached_next_index; + n_left = frame->n_vectors; + next = nexts; - if (node->flags & VLIB_NODE_FLAG_TRACE) - ip6_forward_next_trace (vm, node, frame, VLIB_TX); + vlib_get_buffers (vm, from, bufs, n_left); - while (n_left_from > 0) + while (n_left >= 4) { - vlib_get_next_frame (vm, node, next, to_next, n_left_to_next); + const load_balance_t *lb0, *lb1; + const ip6_header_t *ip0, *ip1; + u32 lbi0, hc0, lbi1, hc1; + const dpo_id_t *dpo0, *dpo1; - - while (n_left_from >= 4 && n_left_to_next >= 2) + /* Prefetch next iteration. */ + { + vlib_prefetch_buffer_header (b[2], STORE); + vlib_prefetch_buffer_header (b[3], STORE); + + CLIB_PREFETCH (b[2]->data, sizeof (ip0[0]), STORE); + CLIB_PREFETCH (b[3]->data, sizeof (ip0[0]), STORE); + } + + ip0 = vlib_buffer_get_current (b[0]); + ip1 = vlib_buffer_get_current (b[1]); + lbi0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX]; + lbi1 = vnet_buffer (b[1])->ip.adj_index[VLIB_TX]; + + lb0 = load_balance_get (lbi0); + lb1 = load_balance_get (lbi1); + + /* + * this node is for via FIBs we can re-use the hash value from the + * to node if present. + * We don't want to use the same hash value at each level in the recursion + * graph as that would lead to polarisation + */ + hc0 = hc1 = 0; + + if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) { - ip_lookup_next_t next0, next1; - const load_balance_t *lb0, *lb1; - vlib_buffer_t *p0, *p1; - u32 pi0, lbi0, hc0, pi1, lbi1, hc1; - const ip6_header_t *ip0, *ip1; - const dpo_id_t *dpo0, *dpo1; - - /* Prefetch next iteration. */ - { - vlib_buffer_t *p2, *p3; - - p2 = vlib_get_buffer (vm, from[2]); - p3 = vlib_get_buffer (vm, from[3]); + if (PREDICT_TRUE (vnet_buffer (b[0])->ip.flow_hash)) + { + hc0 = vnet_buffer (b[0])->ip.flow_hash = + vnet_buffer (b[0])->ip.flow_hash >> 1; + } + else + { + hc0 = vnet_buffer (b[0])->ip.flow_hash = + ip6_compute_flow_hash (ip0, lb0->lb_hash_config); + } + dpo0 = load_balance_get_fwd_bucket + (lb0, (hc0 & (lb0->lb_n_buckets_minus_1))); + } + else + { + dpo0 = load_balance_get_bucket_i (lb0, 0); + } + if (PREDICT_FALSE (lb1->lb_n_buckets > 1)) + { + if (PREDICT_TRUE (vnet_buffer (b[1])->ip.flow_hash)) + { + hc1 = vnet_buffer (b[1])->ip.flow_hash = + vnet_buffer (b[1])->ip.flow_hash >> 1; + } + else + { + hc1 = vnet_buffer (b[1])->ip.flow_hash = + ip6_compute_flow_hash (ip1, lb1->lb_hash_config); + } + dpo1 = load_balance_get_fwd_bucket + (lb1, (hc1 & (lb1->lb_n_buckets_minus_1))); + } + else + { + dpo1 = load_balance_get_bucket_i (lb1, 0); + } - vlib_prefetch_buffer_header (p2, STORE); - vlib_prefetch_buffer_header (p3, STORE); + next[0] = dpo0->dpoi_next_node; + next[1] = dpo1->dpoi_next_node; - CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE); - CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE); - } + /* Only process the HBH Option Header if explicitly configured to do so */ + if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) + { + next[0] = (dpo_is_adj (dpo0) && im->hbh_enabled) ? + (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[0]; + } + /* Only process the HBH Option Header if explicitly configured to do so */ + if (PREDICT_FALSE (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) + { + next[1] = (dpo_is_adj (dpo1) && im->hbh_enabled) ? + (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[1]; + } - pi0 = to_next[0] = from[0]; - pi1 = to_next[1] = from[1]; + vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; + vnet_buffer (b[1])->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; - from += 2; - n_left_from -= 2; - to_next += 2; - n_left_to_next -= 2; + vlib_increment_combined_counter + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b[0])); + vlib_increment_combined_counter + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, b[1])); - p0 = vlib_get_buffer (vm, pi0); - p1 = vlib_get_buffer (vm, pi1); + b += 2; + next += 2; + n_left -= 2; + } - ip0 = vlib_buffer_get_current (p0); - ip1 = vlib_buffer_get_current (p1); - lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX]; - lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX]; + while (n_left > 0) + { + const load_balance_t *lb0; + const ip6_header_t *ip0; + const dpo_id_t *dpo0; + u32 lbi0, hc0; - lb0 = load_balance_get (lbi0); - lb1 = load_balance_get (lbi1); + ip0 = vlib_buffer_get_current (b[0]); + lbi0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX]; - /* - * this node is for via FIBs we can re-use the hash value from the - * to node if present. - * We don't want to use the same hash value at each level in the recursion - * graph as that would lead to polarisation - */ - hc0 = vnet_buffer (p0)->ip.flow_hash = 0; - hc1 = vnet_buffer (p1)->ip.flow_hash = 0; + lb0 = load_balance_get (lbi0); - if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) + hc0 = 0; + if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) + { + if (PREDICT_TRUE (vnet_buffer (b[0])->ip.flow_hash)) { - if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash)) - { - hc0 = vnet_buffer (p0)->ip.flow_hash = - vnet_buffer (p0)->ip.flow_hash >> 1; - } - else - { - hc0 = vnet_buffer (p0)->ip.flow_hash = - ip6_compute_flow_hash (ip0, hc0); - } + hc0 = vnet_buffer (b[0])->ip.flow_hash = + vnet_buffer (b[0])->ip.flow_hash >> 1; } - if (PREDICT_FALSE (lb1->lb_n_buckets > 1)) + else { - if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash)) - { - hc1 = vnet_buffer (p1)->ip.flow_hash = - vnet_buffer (p1)->ip.flow_hash >> 1; - } - else - { - hc1 = vnet_buffer (p1)->ip.flow_hash = - ip6_compute_flow_hash (ip1, hc1); - } + hc0 = vnet_buffer (b[0])->ip.flow_hash = + ip6_compute_flow_hash (ip0, lb0->lb_hash_config); } + dpo0 = load_balance_get_fwd_bucket + (lb0, (hc0 & (lb0->lb_n_buckets_minus_1))); + } + else + { + dpo0 = load_balance_get_bucket_i (lb0, 0); + } - dpo0 = - load_balance_get_bucket_i (lb0, - hc0 & (lb0->lb_n_buckets_minus_1)); - dpo1 = - load_balance_get_bucket_i (lb1, - hc1 & (lb1->lb_n_buckets_minus_1)); + next[0] = dpo0->dpoi_next_node; + vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - next0 = dpo0->dpoi_next_node; - next1 = dpo1->dpoi_next_node; + /* Only process the HBH Option Header if explicitly configured to do so */ + if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) + { + next[0] = (dpo_is_adj (dpo0) && im->hbh_enabled) ? + (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[0]; + } - /* Only process the HBH Option Header if explicitly configured to do so */ - if (PREDICT_FALSE - (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) - { - next0 = (dpo_is_adj (dpo0) && im->hbh_enabled) ? - (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0; - } - /* Only process the HBH Option Header if explicitly configured to do so */ - if (PREDICT_FALSE - (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) - { - next1 = (dpo_is_adj (dpo1) && im->hbh_enabled) ? - (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next1; - } - - vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; - - vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); - vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); - - vlib_validate_buffer_enqueue_x2 (vm, node, next, - to_next, n_left_to_next, - pi0, pi1, next0, next1); - } - - while (n_left_from > 0 && n_left_to_next > 0) - { - ip_lookup_next_t next0; - const load_balance_t *lb0; - vlib_buffer_t *p0; - u32 pi0, lbi0, hc0; - const ip6_header_t *ip0; - const dpo_id_t *dpo0; - - pi0 = from[0]; - to_next[0] = pi0; - from += 1; - to_next += 1; - n_left_to_next -= 1; - n_left_from -= 1; - - p0 = vlib_get_buffer (vm, pi0); - - ip0 = vlib_buffer_get_current (p0); - lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX]; - - lb0 = load_balance_get (lbi0); - - hc0 = vnet_buffer (p0)->ip.flow_hash = 0; - if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) - { - if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash)) - { - hc0 = vnet_buffer (p0)->ip.flow_hash = - vnet_buffer (p0)->ip.flow_hash >> 1; - } - else - { - hc0 = vnet_buffer (p0)->ip.flow_hash = - ip6_compute_flow_hash (ip0, hc0); - } - } - dpo0 = - load_balance_get_bucket_i (lb0, - hc0 & (lb0->lb_n_buckets_minus_1)); - - next0 = dpo0->dpoi_next_node; - vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - - /* Only process the HBH Option Header if explicitly configured to do so */ - if (PREDICT_FALSE - (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) - { - next0 = (dpo_is_adj (dpo0) && im->hbh_enabled) ? - (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0; - } + vlib_increment_combined_counter + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b[0])); - vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + b += 1; + next += 1; + n_left -= 1; + } - vlib_validate_buffer_enqueue_x1 (vm, node, next, - to_next, n_left_to_next, - pi0, next0); - } + vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); - vlib_put_next_frame (vm, node, next, n_left_to_next); - } + if (node->flags & VLIB_NODE_FLAG_TRACE) + ip6_forward_next_trace (vm, node, frame, VLIB_TX); return frame->n_vectors; } @@ -894,7 +738,6 @@ ip6_load_balance (vlib_main_t * vm, /* *INDENT-OFF* */ VLIB_REGISTER_NODE (ip6_load_balance_node) = { - .function = ip6_load_balance, .name = "ip6-load-balance", .vector_size = sizeof (u32), .sibling_of = "ip6-lookup", @@ -902,8 +745,6 @@ VLIB_REGISTER_NODE (ip6_load_balance_node) = }; /* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (ip6_load_balance_node, ip6_load_balance); - typedef struct { /* Adjacency taken. */ @@ -916,19 +757,21 @@ typedef struct } ip6_forward_next_trace_t; -static u8 * +#ifndef CLIB_MARCH_VARIANT +u8 * format_ip6_forward_next_trace (u8 * s, va_list * args) { CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *); - uword indent = format_get_indent (s); + u32 indent = format_get_indent (s); s = format (s, "%U%U", format_white_space, indent, format_ip6_header, t->packet_data, sizeof (t->packet_data)); return s; } +#endif static u8 * format_ip6_lookup_trace (u8 * s, va_list * args) @@ -936,7 +779,7 @@ format_ip6_lookup_trace (u8 * s, va_list * args) CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *); - uword indent = format_get_indent (s); + u32 indent = format_get_indent (s); s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x", t->fib_index, t->adj_index, t->flow_hash); @@ -953,8 +796,7 @@ format_ip6_rewrite_trace (u8 * s, va_list * args) CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *); - vnet_main_t *vnm = vnet_get_main (); - uword indent = format_get_indent (s); + u32 indent = format_get_indent (s); s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x", t->fib_index, t->adj_index, format_ip_adjacency, @@ -962,11 +804,12 @@ format_ip6_rewrite_trace (u8 * s, va_list * args) s = format (s, "\n%U%U", format_white_space, indent, format_ip_adjacency_packet_data, - vnm, t->adj_index, t->packet_data, sizeof (t->packet_data)); + t->adj_index, t->packet_data, sizeof (t->packet_data)); return s; } /* Common trace function for all ip6-forward next nodes. */ +#ifndef CLIB_MARCH_VARIANT void ip6_forward_next_trace (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -1005,9 +848,9 @@ ip6_forward_next_trace (vlib_main_t * vm, vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b0)->sw_if_index[VLIB_RX]); - clib_memcpy (t0->packet_data, - vlib_buffer_get_current (b0), - sizeof (t0->packet_data)); + clib_memcpy_fast (t0->packet_data, + vlib_buffer_get_current (b0), + sizeof (t0->packet_data)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -1020,9 +863,9 @@ ip6_forward_next_trace (vlib_main_t * vm, vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b1)->sw_if_index[VLIB_RX]); - clib_memcpy (t1->packet_data, - vlib_buffer_get_current (b1), - sizeof (t1->packet_data)); + clib_memcpy_fast (t1->packet_data, + vlib_buffer_get_current (b1), + sizeof (t1->packet_data)); } from += 2; n_left -= 2; @@ -1049,95 +892,15 @@ ip6_forward_next_trace (vlib_main_t * vm, vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b0)->sw_if_index[VLIB_RX]); - clib_memcpy (t0->packet_data, - vlib_buffer_get_current (b0), - sizeof (t0->packet_data)); + clib_memcpy_fast (t0->packet_data, + vlib_buffer_get_current (b0), + sizeof (t0->packet_data)); } from += 1; n_left -= 1; } } -static uword -ip6_drop_or_punt (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame, ip6_error_t error_code) -{ - u32 *buffers = vlib_frame_vector_args (frame); - uword n_packets = frame->n_vectors; - - vlib_error_drop_buffers (vm, node, buffers, - /* stride */ 1, - n_packets, - /* next */ 0, - ip6_input_node.index, error_code); - - if (node->flags & VLIB_NODE_FLAG_TRACE) - ip6_forward_next_trace (vm, node, frame, VLIB_TX); - - return n_packets; -} - -static uword -ip6_drop (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_DROP); -} - -static uword -ip6_punt (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_PUNT); -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (ip6_drop_node, static) = -{ - .function = ip6_drop, - .name = "ip6-drop", - .vector_size = sizeof (u32), - .format_trace = format_ip6_forward_next_trace, - .n_next_nodes = 1, - .next_nodes = - { - [0] = "error-drop",}, -}; -/* *INDENT-ON* */ - -VLIB_NODE_FUNCTION_MULTIARCH (ip6_drop_node, ip6_drop); - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (ip6_punt_node, static) = -{ - .function = ip6_punt, - .name = "ip6-punt", - .vector_size = sizeof (u32), - .format_trace = format_ip6_forward_next_trace, - .n_next_nodes = 1, - .next_nodes = - { - [0] = "error-punt",}, -}; -/* *INDENT-ON* */ - -VLIB_NODE_FUNCTION_MULTIARCH (ip6_punt_node, ip6_punt); - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (ip6_multicast_node, static) = -{ - .function = ip6_drop, - .name = "ip6-multicast", - .vector_size = sizeof (u32), - .format_trace = format_ip6_forward_next_trace, - .n_next_nodes = 1, - .next_nodes = - { - [0] = "error-drop", - }, -}; - -/* *INDENT-ON* */ - /* Compute TCP/UDP/ICMP6 checksum in software. */ u16 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, @@ -1169,7 +932,8 @@ ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, uword)); } - /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */ + /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) + * or UDP-Ping packets */ if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) { u32 skip_bytes; @@ -1177,7 +941,8 @@ ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, (ip6_hop_by_hop_ext_t *) data_this_buffer; /* validate really icmp6 next */ - ASSERT (ext_hdr->next_hdr == IP_PROTOCOL_ICMP6); + ASSERT ((ext_hdr->next_hdr == IP_PROTOCOL_ICMP6) + || (ext_hdr->next_hdr == IP_PROTOCOL_UDP)); skip_bytes = 8 * (1 + ext_hdr->n_data_u64s); data_this_buffer = (void *) ((u8 *) data_this_buffer + skip_bytes); @@ -1205,7 +970,7 @@ ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, } p0 = vlib_get_buffer (vm, p0->next_buffer); data_this_buffer = vlib_buffer_get_current (p0); - n_this_buffer = p0->current_length; + n_this_buffer = clib_min (p0->current_length, n_bytes_left); } sum16 = ~ip_csum_fold (sum0); @@ -1230,667 +995,457 @@ ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0) udp0 = (void *) (ip0 + 1); if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0) { - p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED - | IP_BUFFER_L4_CHECKSUM_CORRECT); + p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED + | VNET_BUFFER_F_L4_CHECKSUM_CORRECT); return p0->flags; } sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length); - p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED - | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT)); + p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED + | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT)); return p0->flags; } +#endif -/* ip6_locate_header - * - * This function is to search for the header specified by the find_hdr number. - * 1. If the find_hdr < 0 then it finds and returns the protocol number and - * offset stored in *offset of the transport or ESP header in the chain if - * found. - * 2. If a header with find_hdr > 0 protocol number is found then the - * offset is stored in *offset and protocol number of the header is - * returned. - * 3. If find_hdr header is not found or packet is malformed or - * it is a non-first fragment -1 is returned. +/** + * @brief returns number of links on which src is reachable. */ always_inline int -ip6_locate_header (vlib_buffer_t * p0, - ip6_header_t * ip0, int find_hdr, u32 * offset) -{ - u8 next_proto = ip0->protocol; - u8 *next_header; - u8 done = 0; - u32 cur_offset; - u8 *temp_nxthdr = 0; - u32 exthdr_len = 0; - - next_header = ip6_next_header (ip0); - cur_offset = sizeof (ip6_header_t); - while (1) +ip6_urpf_loose_check (ip6_main_t * im, vlib_buffer_t * b, ip6_header_t * i) +{ + const load_balance_t *lb0; + index_t lbi; + u32 fib_index; + + fib_index = vec_elt (im->fib_index_by_sw_if_index, + vnet_buffer (b)->sw_if_index[VLIB_RX]); + fib_index = + (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ? + fib_index : vnet_buffer (b)->sw_if_index[VLIB_TX]; + + lbi = ip6_fib_table_fwding_lookup (im, fib_index, &i->src_address); + lb0 = load_balance_get (lbi); + + return (fib_urpf_check_size (lb0->lb_urpf)); +} + +always_inline u8 +ip6_next_proto_is_tcp_udp (vlib_buffer_t * p0, ip6_header_t * ip0, + u32 * udp_offset0) +{ + u32 proto0; + proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_UDP, udp_offset0); + if (proto0 != IP_PROTOCOL_UDP) { - done = (next_proto == find_hdr); - if (PREDICT_FALSE - (next_header >= - (u8 *) vlib_buffer_get_current (p0) + p0->current_length)) - { - //A malicious packet could set an extension header with a too big size - return (-1); - } - if (done) - break; - if ((!ip6_ext_hdr (next_proto)) || next_proto == IP_PROTOCOL_IP6_NONXT) - { - if (find_hdr < 0) - break; - return -1; - } - if (next_proto == IP_PROTOCOL_IPV6_FRAGMENTATION) - { - ip6_frag_hdr_t *frag_hdr = (ip6_frag_hdr_t *) next_header; - u16 frag_off = ip6_frag_hdr_offset (frag_hdr); - /* Non first fragment return -1 */ - if (frag_off) - return (-1); - exthdr_len = sizeof (ip6_frag_hdr_t); - temp_nxthdr = next_header + exthdr_len; - } - else if (next_proto == IP_PROTOCOL_IPSEC_AH) - { - exthdr_len = - ip6_ext_authhdr_len (((ip6_ext_header_t *) next_header)); - temp_nxthdr = next_header + exthdr_len; - } - else - { - exthdr_len = - ip6_ext_header_len (((ip6_ext_header_t *) next_header)); - temp_nxthdr = next_header + exthdr_len; - } - next_proto = ((ip6_ext_header_t *) next_header)->next_hdr; - next_header = temp_nxthdr; - cur_offset += exthdr_len; + proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_TCP, udp_offset0); + proto0 = (proto0 == IP_PROTOCOL_TCP) ? proto0 : 0; } - - *offset = cur_offset; - return (next_proto); + return proto0; } -static uword -ip6_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) +/* *INDENT-OFF* */ +VNET_FEATURE_ARC_INIT (ip6_local) = +{ + .arc_name = "ip6-local", + .start_nodes = VNET_FEATURES ("ip6-local"), +}; +/* *INDENT-ON* */ + +always_inline uword +ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * frame, int head_of_feature_arc) { ip6_main_t *im = &ip6_main; ip_lookup_main_t *lm = &im->lookup_main; - ip_local_next_t next_index; - u32 *from, *to_next, n_left_from, n_left_to_next; + u32 *from, n_left_from; vlib_node_runtime_t *error_node = vlib_node_get_runtime (vm, ip6_input_node.index); + u8 arc_index = vnet_feat_arc_ip6_local.feature_arc_index; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; + u16 nexts[VLIB_FRAME_SIZE], *next; from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; - next_index = node->cached_next_index; if (node->flags & VLIB_NODE_FLAG_TRACE) ip6_forward_next_trace (vm, node, frame, VLIB_TX); - while (n_left_from > 0) - { - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + vlib_get_buffers (vm, from, bufs, n_left_from); + b = bufs; + next = nexts; - while (n_left_from >= 4 && n_left_to_next >= 2) + while (n_left_from > 2) + { + /* Prefetch next iteration. */ + if (n_left_from >= 6) { - vlib_buffer_t *p0, *p1; - ip6_header_t *ip0, *ip1; - udp_header_t *udp0, *udp1; - u32 pi0, ip_len0, udp_len0, flags0, next0; - u32 pi1, ip_len1, udp_len1, flags1, next1; - i32 len_diff0, len_diff1; - u8 error0, type0, good_l4_checksum0; - u8 error1, type1, good_l4_checksum1; - u32 udp_offset0, udp_offset1; - - pi0 = to_next[0] = from[0]; - pi1 = to_next[1] = from[1]; - from += 2; - n_left_from -= 2; - to_next += 2; - n_left_to_next -= 2; - - p0 = vlib_get_buffer (vm, pi0); - p1 = vlib_get_buffer (vm, pi1); - - ip0 = vlib_buffer_get_current (p0); - ip1 = vlib_buffer_get_current (p1); - - vnet_buffer (p0)->ip.start_of_ip_header = p0->current_data; - vnet_buffer (p1)->ip.start_of_ip_header = p1->current_data; - - type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol]; - type1 = lm->builtin_protocol_by_ip_protocol[ip1->protocol]; - - next0 = lm->local_next_by_ip_protocol[ip0->protocol]; - next1 = lm->local_next_by_ip_protocol[ip1->protocol]; + vlib_prefetch_buffer_header (b[4], STORE); + vlib_prefetch_buffer_header (b[5], STORE); + vlib_prefetch_buffer_data (b[2], LOAD); + vlib_prefetch_buffer_data (b[3], LOAD); + } - flags0 = p0->flags; - flags1 = p1->flags; + u8 error[2]; + error[0] = IP6_ERROR_UNKNOWN_PROTOCOL; + error[1] = IP6_ERROR_UNKNOWN_PROTOCOL; - good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0; - good_l4_checksum1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0; - len_diff0 = 0; - len_diff1 = 0; + ip6_header_t *ip[2]; + ip[0] = vlib_buffer_get_current (b[0]); + ip[1] = vlib_buffer_get_current (b[1]); - /* Skip HBH local processing */ - if (PREDICT_FALSE - (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) - { - ip6_hop_by_hop_ext_t *ext_hdr = - (ip6_hop_by_hop_ext_t *) ip6_next_header (ip0); - next0 = lm->local_next_by_ip_protocol[ext_hdr->next_hdr]; - type0 = lm->builtin_protocol_by_ip_protocol[ext_hdr->next_hdr]; - } - if (PREDICT_FALSE - (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) + if (head_of_feature_arc) + { + vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data; + vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data; + + u8 type[2]; + type[0] = lm->builtin_protocol_by_ip_protocol[ip[0]->protocol]; + type[1] = lm->builtin_protocol_by_ip_protocol[ip[1]->protocol]; + + u32 flags[2]; + flags[0] = b[0]->flags; + flags[1] = b[1]->flags; + + u32 good_l4_csum[2]; + good_l4_csum[0] = + flags[0] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | + VNET_BUFFER_F_OFFLOAD_UDP_CKSUM); + good_l4_csum[1] = + flags[1] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | + VNET_BUFFER_F_OFFLOAD_UDP_CKSUM); + + u32 udp_offset[2] = { }; + u8 is_tcp_udp[2]; + is_tcp_udp[0] = + ip6_next_proto_is_tcp_udp (b[0], ip[0], &udp_offset[0]); + is_tcp_udp[1] = + ip6_next_proto_is_tcp_udp (b[1], ip[1], &udp_offset[1]); + i16 len_diff[2] = { 0 }; + if (PREDICT_TRUE (is_tcp_udp[0])) { - ip6_hop_by_hop_ext_t *ext_hdr = - (ip6_hop_by_hop_ext_t *) ip6_next_header (ip1); - next1 = lm->local_next_by_ip_protocol[ext_hdr->next_hdr]; - type1 = lm->builtin_protocol_by_ip_protocol[ext_hdr->next_hdr]; + udp_header_t *udp = + (udp_header_t *) ((u8 *) ip[0] + udp_offset[0]); + good_l4_csum[0] |= type[0] == IP_BUILTIN_PROTOCOL_UDP + && udp->checksum == 0; + /* optimistically verify UDP length. */ + u16 ip_len, udp_len; + ip_len = clib_net_to_host_u16 (ip[0]->payload_length); + udp_len = clib_net_to_host_u16 (udp->length); + len_diff[0] = ip_len - udp_len; } - if (PREDICT_TRUE (IP_PROTOCOL_UDP == ip6_locate_header (p0, ip0, - IP_PROTOCOL_UDP, - &udp_offset0))) + if (PREDICT_TRUE (is_tcp_udp[1])) { - udp0 = (udp_header_t *) ((u8 *) ip0 + udp_offset0); - /* Don't verify UDP checksum for packets with explicit zero checksum. */ - good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP - && udp0->checksum == 0; - /* Verify UDP length. */ - ip_len0 = clib_net_to_host_u16 (ip0->payload_length); - udp_len0 = clib_net_to_host_u16 (udp0->length); - len_diff0 = ip_len0 - udp_len0; - } - if (PREDICT_TRUE (IP_PROTOCOL_UDP == ip6_locate_header (p1, ip1, - IP_PROTOCOL_UDP, - &udp_offset1))) - { - udp1 = (udp_header_t *) ((u8 *) ip1 + udp_offset1); - /* Don't verify UDP checksum for packets with explicit zero checksum. */ - good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UDP - && udp1->checksum == 0; - /* Verify UDP length. */ - ip_len1 = clib_net_to_host_u16 (ip1->payload_length); - udp_len1 = clib_net_to_host_u16 (udp1->length); - len_diff1 = ip_len1 - udp_len1; + udp_header_t *udp = + (udp_header_t *) ((u8 *) ip[1] + udp_offset[1]); + good_l4_csum[1] |= type[1] == IP_BUILTIN_PROTOCOL_UDP + && udp->checksum == 0; + /* optimistically verify UDP length. */ + u16 ip_len, udp_len; + ip_len = clib_net_to_host_u16 (ip[1]->payload_length); + udp_len = clib_net_to_host_u16 (udp->length); + len_diff[1] = ip_len - udp_len; } - good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN; - good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UNKNOWN; + good_l4_csum[0] |= type[0] == IP_BUILTIN_PROTOCOL_UNKNOWN; + good_l4_csum[1] |= type[1] == IP_BUILTIN_PROTOCOL_UNKNOWN; - len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0; - len_diff1 = type1 == IP_BUILTIN_PROTOCOL_UDP ? len_diff1 : 0; + len_diff[0] = type[0] == IP_BUILTIN_PROTOCOL_UDP ? len_diff[0] : 0; + len_diff[1] = type[1] == IP_BUILTIN_PROTOCOL_UDP ? len_diff[1] : 0; - if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN - && !good_l4_checksum0 - && !(flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))) + u8 need_csum[2]; + need_csum[0] = type[0] != IP_BUILTIN_PROTOCOL_UNKNOWN + && !good_l4_csum[0] + && !(flags[0] & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + need_csum[1] = type[1] != IP_BUILTIN_PROTOCOL_UNKNOWN + && !good_l4_csum[1] + && !(flags[1] & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + if (PREDICT_FALSE (need_csum[0])) { - flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0); - good_l4_checksum0 = - (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0; + flags[0] = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]); + good_l4_csum[0] = flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT; } - if (PREDICT_FALSE (type1 != IP_BUILTIN_PROTOCOL_UNKNOWN - && !good_l4_checksum1 - && !(flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED))) + if (PREDICT_FALSE (need_csum[1])) { - flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, p1); - good_l4_checksum1 = - (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0; + flags[1] = ip6_tcp_udp_icmp_validate_checksum (vm, b[1]); + good_l4_csum[1] = flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT; } - error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL; + error[0] = IP6_ERROR_UNKNOWN_PROTOCOL; + error[0] = len_diff[0] < 0 ? IP6_ERROR_UDP_LENGTH : error[0]; + error[1] = IP6_ERROR_UNKNOWN_PROTOCOL; + error[1] = len_diff[1] < 0 ? IP6_ERROR_UDP_LENGTH : error[1]; - error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0; - error1 = len_diff1 < 0 ? IP6_ERROR_UDP_LENGTH : error1; + STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == + IP6_ERROR_UDP_CHECKSUM, + "Wrong IP6 errors constants"); + STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == + IP6_ERROR_ICMP_CHECKSUM, + "Wrong IP6 errors constants"); - ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == - IP6_ERROR_UDP_CHECKSUM); - ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == - IP6_ERROR_ICMP_CHECKSUM); - error0 = - (!good_l4_checksum0 ? IP6_ERROR_UDP_CHECKSUM + type0 : error0); - error1 = - (!good_l4_checksum1 ? IP6_ERROR_UDP_CHECKSUM + type1 : error1); + error[0] = + !good_l4_csum[0] ? IP6_ERROR_UDP_CHECKSUM + type[0] : error[0]; + error[1] = + !good_l4_csum[1] ? IP6_ERROR_UDP_CHECKSUM + type[1] : error[1]; /* Drop packets from unroutable hosts. */ /* If this is a neighbor solicitation (ICMP), skip source RPF check */ - if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL && - type0 != IP_BUILTIN_PROTOCOL_ICMP && - !ip6_address_is_link_local_unicast (&ip0->src_address)) + u8 unroutable[2]; + unroutable[0] = error[0] == IP6_ERROR_UNKNOWN_PROTOCOL + && type[0] != IP_BUILTIN_PROTOCOL_ICMP + && !ip6_address_is_link_local_unicast (&ip[0]->src_address); + unroutable[1] = error[1] == IP6_ERROR_UNKNOWN_PROTOCOL + && type[1] != IP_BUILTIN_PROTOCOL_ICMP + && !ip6_address_is_link_local_unicast (&ip[1]->src_address); + if (PREDICT_FALSE (unroutable[0])) { - u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0); - error0 = (ADJ_INDEX_INVALID == src_adj_index0 - ? IP6_ERROR_SRC_LOOKUP_MISS : error0); + error[0] = + !ip6_urpf_loose_check (im, b[0], + ip[0]) ? IP6_ERROR_SRC_LOOKUP_MISS + : error[0]; } - if (error1 == IP6_ERROR_UNKNOWN_PROTOCOL && - type1 != IP_BUILTIN_PROTOCOL_ICMP && - !ip6_address_is_link_local_unicast (&ip1->src_address)) + if (PREDICT_FALSE (unroutable[1])) { - u32 src_adj_index1 = ip6_src_lookup_for_packet (im, p1, ip1); - error1 = (ADJ_INDEX_INVALID == src_adj_index1 - ? IP6_ERROR_SRC_LOOKUP_MISS : error1); + error[1] = + !ip6_urpf_loose_check (im, b[1], + ip[1]) ? IP6_ERROR_SRC_LOOKUP_MISS + : error[1]; } - next0 = - error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0; - next1 = - error1 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1; - - p0->error = error_node->errors[error0]; - p1->error = error_node->errors[error1]; - - vlib_validate_buffer_enqueue_x2 (vm, node, next_index, - to_next, n_left_to_next, - pi0, pi1, next0, next1); - } - - while (n_left_from > 0 && n_left_to_next > 0) + vnet_buffer (b[0])->ip.fib_index = + vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ? + vnet_buffer (b[0])->sw_if_index[VLIB_TX] : + vnet_buffer (b[0])->ip.fib_index; + vnet_buffer (b[1])->ip.fib_index = + vnet_buffer (b[1])->sw_if_index[VLIB_TX] != ~0 ? + vnet_buffer (b[1])->sw_if_index[VLIB_TX] : + vnet_buffer (b[1])->ip.fib_index; + } /* head_of_feature_arc */ + + next[0] = lm->local_next_by_ip_protocol[ip[0]->protocol]; + next[0] = + error[0] != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[0]; + next[1] = lm->local_next_by_ip_protocol[ip[1]->protocol]; + next[1] = + error[1] != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[1]; + + b[0]->error = error_node->errors[0]; + b[1]->error = error_node->errors[1]; + + if (head_of_feature_arc) { - vlib_buffer_t *p0; - ip6_header_t *ip0; - udp_header_t *udp0; - u32 pi0, ip_len0, udp_len0, flags0, next0; - i32 len_diff0; - u8 error0, type0, good_l4_checksum0; - u32 udp_offset0; - - pi0 = to_next[0] = from[0]; - from += 1; - n_left_from -= 1; - to_next += 1; - n_left_to_next -= 1; - - p0 = vlib_get_buffer (vm, pi0); - - ip0 = vlib_buffer_get_current (p0); - - vnet_buffer (p0)->ip.start_of_ip_header = p0->current_data; - - type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol]; - next0 = lm->local_next_by_ip_protocol[ip0->protocol]; - - flags0 = p0->flags; - - good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0; - len_diff0 = 0; - - /* Skip HBH local processing */ - if (PREDICT_FALSE - (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS)) + u8 ip6_unknown[2]; + ip6_unknown[0] = error[0] == (u8) IP6_ERROR_UNKNOWN_PROTOCOL; + ip6_unknown[1] = error[1] == (u8) IP6_ERROR_UNKNOWN_PROTOCOL; + if (PREDICT_TRUE (ip6_unknown[0])) { - ip6_hop_by_hop_ext_t *ext_hdr = - (ip6_hop_by_hop_ext_t *) ip6_next_header (ip0); - next0 = lm->local_next_by_ip_protocol[ext_hdr->next_hdr]; - type0 = lm->builtin_protocol_by_ip_protocol[ext_hdr->next_hdr]; + u32 next32 = next[0]; + vnet_feature_arc_start (arc_index, + vnet_buffer (b[0])->sw_if_index + [VLIB_RX], &next32, b[0]); + next[0] = next32; } - if (PREDICT_TRUE (IP_PROTOCOL_UDP == ip6_locate_header (p0, ip0, - IP_PROTOCOL_UDP, - &udp_offset0))) + if (PREDICT_TRUE (ip6_unknown[1])) { - udp0 = (udp_header_t *) ((u8 *) ip0 + udp_offset0); - /* Don't verify UDP checksum for packets with explicit zero checksum. */ - good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP - && udp0->checksum == 0; - /* Verify UDP length. */ - ip_len0 = clib_net_to_host_u16 (ip0->payload_length); - udp_len0 = clib_net_to_host_u16 (udp0->length); - len_diff0 = ip_len0 - udp_len0; + u32 next32 = next[1]; + vnet_feature_arc_start (arc_index, + vnet_buffer (b[1])->sw_if_index + [VLIB_RX], &next32, b[1]); + next[1] = next32; } - - good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN; - len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0; - - if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN - && !good_l4_checksum0 - && !(flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))) - { - flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0); - good_l4_checksum0 = - (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0; - } - - error0 = IP6_ERROR_UNKNOWN_PROTOCOL; - - error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0; - - ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == - IP6_ERROR_UDP_CHECKSUM); - ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == - IP6_ERROR_ICMP_CHECKSUM); - error0 = - (!good_l4_checksum0 ? IP6_ERROR_UDP_CHECKSUM + type0 : error0); - - /* If this is a neighbor solicitation (ICMP), skip source RPF check */ - if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL && - type0 != IP_BUILTIN_PROTOCOL_ICMP && - !ip6_address_is_link_local_unicast (&ip0->src_address)) - { - u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0); - error0 = (ADJ_INDEX_INVALID == src_adj_index0 - ? IP6_ERROR_SRC_LOOKUP_MISS : error0); - } - - next0 = - error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0; - - p0->error = error_node->errors[error0]; - - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - pi0, next0); } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); + /* next */ + b += 2; + next += 2; + n_left_from -= 2; } - return frame->n_vectors; -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (ip6_local_node, static) = -{ - .function = ip6_local, - .name = "ip6-local", - .vector_size = sizeof (u32), - .format_trace = format_ip6_forward_next_trace, - .n_next_nodes = IP_LOCAL_N_NEXT, - .next_nodes = - { - [IP_LOCAL_NEXT_DROP] = "error-drop", - [IP_LOCAL_NEXT_PUNT] = "error-punt", - [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup", - [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input", - }, -}; -/* *INDENT-ON* */ - -VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_node, ip6_local); - -void -ip6_register_protocol (u32 protocol, u32 node_index) -{ - vlib_main_t *vm = vlib_get_main (); - ip6_main_t *im = &ip6_main; - ip_lookup_main_t *lm = &im->lookup_main; - - ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol)); - lm->local_next_by_ip_protocol[protocol] = - vlib_node_add_next (vm, ip6_local_node.index, node_index); -} - -typedef enum -{ - IP6_DISCOVER_NEIGHBOR_NEXT_DROP, - IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX, - IP6_DISCOVER_NEIGHBOR_N_NEXT, -} ip6_discover_neighbor_next_t; - -typedef enum -{ - IP6_DISCOVER_NEIGHBOR_ERROR_DROP, - IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT, - IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS, -} ip6_discover_neighbor_error_t; - -static uword -ip6_discover_neighbor_inline (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame, int is_glean) -{ - vnet_main_t *vnm = vnet_get_main (); - ip6_main_t *im = &ip6_main; - ip_lookup_main_t *lm = &im->lookup_main; - u32 *from, *to_next_drop; - uword n_left_from, n_left_to_next_drop; - static f64 time_last_seed_change = -1e100; - static u32 hash_seeds[3]; - static uword hash_bitmap[256 / BITS (uword)]; - f64 time_now; - int bogus_length; - - if (node->flags & VLIB_NODE_FLAG_TRACE) - ip6_forward_next_trace (vm, node, frame, VLIB_TX); - - time_now = vlib_time_now (vm); - if (time_now - time_last_seed_change > 1e-3) + while (n_left_from) { - uword i; - u32 *r = clib_random_buffer_get_data (&vm->random_buffer, - sizeof (hash_seeds)); - for (i = 0; i < ARRAY_LEN (hash_seeds); i++) - hash_seeds[i] = r[i]; - - /* Mark all hash keys as been not-seen before. */ - for (i = 0; i < ARRAY_LEN (hash_bitmap); i++) - hash_bitmap[i] = 0; - - time_last_seed_change = time_now; - } - - from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - - while (n_left_from > 0) - { - vlib_get_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP, - to_next_drop, n_left_to_next_drop); - - while (n_left_from > 0 && n_left_to_next_drop > 0) - { - vlib_buffer_t *p0; - ip6_header_t *ip0; - u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0; - uword bm0; - ip_adjacency_t *adj0; - vnet_hw_interface_t *hw_if0; - u32 next0; - - pi0 = from[0]; - - p0 = vlib_get_buffer (vm, pi0); - - adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX]; - - ip0 = vlib_buffer_get_current (p0); - - adj0 = ip_get_adjacency (lm, adj_index0); - - if (!is_glean) - { - ip0->dst_address.as_u64[0] = - adj0->sub_type.nbr.next_hop.ip6.as_u64[0]; - ip0->dst_address.as_u64[1] = - adj0->sub_type.nbr.next_hop.ip6.as_u64[1]; - } - - a0 = hash_seeds[0]; - b0 = hash_seeds[1]; - c0 = hash_seeds[2]; - - sw_if_index0 = adj0->rewrite_header.sw_if_index; - vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0; - - a0 ^= sw_if_index0; - b0 ^= ip0->dst_address.as_u32[0]; - c0 ^= ip0->dst_address.as_u32[1]; + u8 error; + error = IP6_ERROR_UNKNOWN_PROTOCOL; - hash_v3_mix32 (a0, b0, c0); + ip6_header_t *ip; + ip = vlib_buffer_get_current (b[0]); - b0 ^= ip0->dst_address.as_u32[2]; - c0 ^= ip0->dst_address.as_u32[3]; + if (head_of_feature_arc) + { + vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data; + u8 type = lm->builtin_protocol_by_ip_protocol[ip->protocol]; + + u32 flags = b[0]->flags; + u32 good_l4_csum = + flags & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT | + VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | + VNET_BUFFER_F_OFFLOAD_UDP_CKSUM); + + u32 udp_offset; + i16 len_diff = 0; + u8 is_tcp_udp = ip6_next_proto_is_tcp_udp (b[0], ip, &udp_offset); + if (PREDICT_TRUE (is_tcp_udp)) + { + udp_header_t *udp = (udp_header_t *) ((u8 *) ip + udp_offset); + /* Don't verify UDP checksum for packets with explicit zero checksum. */ + good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UDP + && udp->checksum == 0; + /* optimistically verify UDP length. */ + u16 ip_len, udp_len; + ip_len = clib_net_to_host_u16 (ip->payload_length); + udp_len = clib_net_to_host_u16 (udp->length); + len_diff = ip_len - udp_len; + } - hash_v3_finalize32 (a0, b0, c0); + good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UNKNOWN; + len_diff = type == IP_BUILTIN_PROTOCOL_UDP ? len_diff : 0; - c0 &= BITS (hash_bitmap) - 1; - c0 = c0 / BITS (uword); - m0 = (uword) 1 << (c0 % BITS (uword)); + u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN && !good_l4_csum + && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + if (PREDICT_FALSE (need_csum)) + { + flags = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]); + good_l4_csum = flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT; + } - bm0 = hash_bitmap[c0]; - drop0 = (bm0 & m0) != 0; + error = IP6_ERROR_UNKNOWN_PROTOCOL; + error = len_diff < 0 ? IP6_ERROR_UDP_LENGTH : error; - /* Mark it as seen. */ - hash_bitmap[c0] = bm0 | m0; + STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == + IP6_ERROR_UDP_CHECKSUM, + "Wrong IP6 errors constants"); + STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == + IP6_ERROR_ICMP_CHECKSUM, + "Wrong IP6 errors constants"); - from += 1; - n_left_from -= 1; - to_next_drop[0] = pi0; - to_next_drop += 1; - n_left_to_next_drop -= 1; + error = !good_l4_csum ? IP6_ERROR_UDP_CHECKSUM + type : error; - hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); + /* Drop packets from unroutable hosts. */ + /* If this is a neighbor solicitation (ICMP), skip source RPF check */ + u8 unroutable = error == IP6_ERROR_UNKNOWN_PROTOCOL + && type != IP_BUILTIN_PROTOCOL_ICMP + && !ip6_address_is_link_local_unicast (&ip->src_address); + if (PREDICT_FALSE (unroutable)) + { + error = + !ip6_urpf_loose_check (im, b[0], + ip) ? IP6_ERROR_SRC_LOOKUP_MISS : + error; + } - /* If the interface is link-down, drop the pkt */ - if (!(hw_if0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP)) - drop0 = 1; + vnet_buffer (b[0])->ip.fib_index = + vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ? + vnet_buffer (b[0])->sw_if_index[VLIB_TX] : + vnet_buffer (b[0])->ip.fib_index; + } /* head_of_feature_arc */ - p0->error = - node->errors[drop0 ? IP6_DISCOVER_NEIGHBOR_ERROR_DROP - : IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT]; - if (drop0) - continue; + next[0] = lm->local_next_by_ip_protocol[ip->protocol]; + next[0] = + error != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[0]; - /* - * the adj has been updated to a rewrite but the node the DPO that got - * us here hasn't - yet. no big deal. we'll drop while we wait. - */ - if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index) - continue; + b[0]->error = error_node->errors[0]; - { - u32 bi0 = 0; - icmp6_neighbor_solicitation_header_t *h0; - vlib_buffer_t *b0; - - h0 = vlib_packet_template_get_packet - (vm, &im->discover_neighbor_packet_template, &bi0); - - /* - * Build ethernet header. - * Choose source address based on destination lookup - * adjacency. - */ - if (ip6_src_address_for_packet (lm, - sw_if_index0, - &h0->ip.src_address)) - { - /* There is no address on the interface */ - p0->error = - node->errors[IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS]; - vlib_buffer_free (vm, &bi0, 1); - continue; - } - - /* - * Destination address is a solicited node multicast address. - * We need to fill in - * the low 24 bits with low 24 bits of target's address. - */ - h0->ip.dst_address.as_u8[13] = ip0->dst_address.as_u8[13]; - h0->ip.dst_address.as_u8[14] = ip0->dst_address.as_u8[14]; - h0->ip.dst_address.as_u8[15] = ip0->dst_address.as_u8[15]; - - h0->neighbor.target_address = ip0->dst_address; - - clib_memcpy (h0->link_layer_option.ethernet_address, - hw_if0->hw_address, vec_len (hw_if0->hw_address)); - - /* $$$$ appears we need this; why is the checksum non-zero? */ - h0->neighbor.icmp.checksum = 0; - h0->neighbor.icmp.checksum = - ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h0->ip, - &bogus_length); - - ASSERT (bogus_length == 0); - - vlib_buffer_copy_trace_flag (vm, p0, bi0); - b0 = vlib_get_buffer (vm, bi0); - vnet_buffer (b0)->sw_if_index[VLIB_TX] - = vnet_buffer (p0)->sw_if_index[VLIB_TX]; - - /* Add rewrite/encap string. */ - vnet_rewrite_one_header (adj0[0], h0, sizeof (ethernet_header_t)); - vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes); - - next0 = IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX; - - vlib_set_next_frame_buffer (vm, node, next0, bi0); - } + if (head_of_feature_arc) + { + if (PREDICT_TRUE (error == (u8) IP6_ERROR_UNKNOWN_PROTOCOL)) + { + u32 next32 = next[0]; + vnet_feature_arc_start (arc_index, + vnet_buffer (b[0])->sw_if_index + [VLIB_RX], &next32, b[0]); + next[0] = next32; + } } - vlib_put_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP, - n_left_to_next_drop); + /* next */ + b += 1; + next += 1; + n_left_from -= 1; } + vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); return frame->n_vectors; } -static uword -ip6_discover_neighbor (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) -{ - return (ip6_discover_neighbor_inline (vm, node, frame, 0)); -} - -static uword -ip6_glean (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) +VLIB_NODE_FN (ip6_local_node) (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * frame) { - return (ip6_discover_neighbor_inline (vm, node, frame, 1)); + return ip6_local_inline (vm, node, frame, 1 /* head of feature arc */ ); } -static char *ip6_discover_neighbor_error_strings[] = { - [IP6_DISCOVER_NEIGHBOR_ERROR_DROP] = "address overflow drops", - [IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT] = "neighbor solicitations sent", - [IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS] - = "no source address for ND solicitation", -}; - /* *INDENT-OFF* */ -VLIB_REGISTER_NODE (ip6_discover_neighbor_node) = +VLIB_REGISTER_NODE (ip6_local_node) = { - .function = ip6_discover_neighbor, - .name = "ip6-discover-neighbor", + .name = "ip6-local", .vector_size = sizeof (u32), .format_trace = format_ip6_forward_next_trace, - .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings), - .error_strings = ip6_discover_neighbor_error_strings, - .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT, + .n_next_nodes = IP_LOCAL_N_NEXT, .next_nodes = { - [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop", - [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output", + [IP_LOCAL_NEXT_DROP] = "ip6-drop", + [IP_LOCAL_NEXT_PUNT] = "ip6-punt", + [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup", + [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input", + [IP_LOCAL_NEXT_REASSEMBLY] = "ip6-reassembly", }, }; /* *INDENT-ON* */ -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (ip6_glean_node) = +VLIB_NODE_FN (ip6_local_end_of_arc_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { - .function = ip6_glean, - .name = "ip6-glean", + return ip6_local_inline (vm, node, frame, 0 /* head of feature arc */ ); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (ip6_local_end_of_arc_node) = { + .name = "ip6-local-end-of-arc", .vector_size = sizeof (u32), + .format_trace = format_ip6_forward_next_trace, - .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings), - .error_strings = ip6_discover_neighbor_error_strings, - .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT, - .next_nodes = - { - [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop", - [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output", - }, + .sibling_of = "ip6-local", +}; + +VNET_FEATURE_INIT (ip6_local_end_of_arc, static) = { + .arc_name = "ip6-local", + .node_name = "ip6-local-end-of-arc", + .runs_before = 0, /* not before any other features */ }; /* *INDENT-ON* */ +#ifdef CLIB_MARCH_VARIANT +extern vlib_node_registration_t ip6_local_node; + +#else + +void +ip6_register_protocol (u32 protocol, u32 node_index) +{ + vlib_main_t *vm = vlib_get_main (); + ip6_main_t *im = &ip6_main; + ip_lookup_main_t *lm = &im->lookup_main; + + ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol)); + lm->local_next_by_ip_protocol[protocol] = + vlib_node_add_next (vm, ip6_local_node.index, node_index); +} + +void +ip6_unregister_protocol (u32 protocol) +{ + ip6_main_t *im = &ip6_main; + ip_lookup_main_t *lm = &im->lookup_main; + + ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol)); + lm->local_next_by_ip_protocol[protocol] = IP_LOCAL_NEXT_PUNT; +} + clib_error_t * -ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index) +ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index, + u8 refresh) { vnet_main_t *vnm = vnet_get_main (); ip6_main_t *im = &ip6_main; @@ -1901,6 +1456,7 @@ ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index) vnet_hw_interface_t *hi; vnet_sw_interface_t *si; vlib_buffer_t *b; + adj_index_t ai; u32 bi = 0; int bogus_length; @@ -1929,6 +1485,8 @@ ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index) vlib_packet_template_get_packet (vm, &im->discover_neighbor_packet_template, &bi); + if (!h) + return clib_error_return (0, "ICMP6 NS packet allocation failed"); hi = vnet_get_sup_hw_interface (vnm, sw_if_index); @@ -1941,8 +1499,16 @@ ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index) h->ip.src_address = src[0]; h->neighbor.target_address = dst[0]; - clib_memcpy (h->link_layer_option.ethernet_address, hi->hw_address, - vec_len (hi->hw_address)); + if (PREDICT_FALSE (!hi->hw_address)) + { + return clib_error_return (0, "%U: interface %U do not support ip probe", + format_ip6_address, dst, + format_vnet_sw_if_index_name, vnm, + sw_if_index); + } + + clib_memcpy_fast (h->link_layer_option.ethernet_address, hi->hw_address, + vec_len (hi->hw_address)); h->neighbor.icmp.checksum = ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length); @@ -1953,7 +1519,23 @@ ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index) vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index; /* Add encapsulation string for software interface (e.g. ethernet header). */ - adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index); + ip46_address_t nh = { + .ip6 = *dst, + }; + + ai = adj_nbr_add_or_lock (FIB_PROTOCOL_IP6, + VNET_LINK_IP6, &nh, sw_if_index); + adj = adj_get (ai); + + /* Peer has been previously resolved, retrieve glean adj instead */ + if (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE && refresh == 0) + { + adj_unlock (ai); + ai = adj_glean_add_or_lock (FIB_PROTOCOL_IP6, + VNET_LINK_IP6, sw_if_index, &nh); + adj = adj_get (ai); + } + vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t)); vlib_buffer_advance (b, -adj->rewrite_header.data_bytes); @@ -1965,19 +1547,56 @@ ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index) vlib_put_frame_to_node (vm, hi->output_node_index, f); } + adj_unlock (ai); return /* no error */ 0; } +#endif typedef enum { IP6_REWRITE_NEXT_DROP, IP6_REWRITE_NEXT_ICMP_ERROR, + IP6_REWRITE_NEXT_FRAGMENT, + IP6_REWRITE_N_NEXT /* Last */ } ip6_rewrite_next_t; +/** + * This bits of an IPv6 address to mask to construct a multicast + * MAC address + */ +#define IP6_MCAST_ADDR_MASK 0xffffffff + +always_inline void +ip6_mtu_check (vlib_buffer_t * b, u16 packet_bytes, + u16 adj_packet_bytes, bool is_locally_generated, + u32 * next, u32 * error) +{ + if (adj_packet_bytes >= 1280 && packet_bytes > adj_packet_bytes) + { + if (is_locally_generated) + { + /* IP fragmentation */ + ip_frag_set_vnet_buffer (b, adj_packet_bytes, + IP6_FRAG_NEXT_IP6_REWRITE, 0); + *next = IP6_REWRITE_NEXT_FRAGMENT; + *error = IP6_ERROR_MTU_EXCEEDED; + } + else + { + *error = IP6_ERROR_MTU_EXCEEDED; + icmp6_error_set_vnet_buffer (b, ICMP6_packet_too_big, 0, + adj_packet_bytes); + *next = IP6_REWRITE_NEXT_ICMP_ERROR; + } + } +} + always_inline uword -ip6_rewrite_inline (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame, int is_midchain) +ip6_rewrite_inline_with_gso (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame, + int do_counters, int is_midchain, int is_mcast, + int do_gso) { ip_lookup_main_t *lm = &ip6_main.lookup_main; u32 *from = vlib_frame_vector_args (frame); @@ -1987,7 +1606,7 @@ ip6_rewrite_inline (vlib_main_t * vm, n_left_from = frame->n_vectors; next_index = node->cached_next_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vm->thread_index; while (n_left_from > 0) { @@ -2001,6 +1620,7 @@ ip6_rewrite_inline (vlib_main_t * vm, u32 pi0, rw_len0, next0, error0, adj_index0; u32 pi1, rw_len1, next1, error1, adj_index1; u32 tx_sw_if_index0, tx_sw_if_index1; + bool is_locally_originated0, is_locally_originated1; /* Prefetch next iteration. */ { @@ -2033,16 +1653,15 @@ ip6_rewrite_inline (vlib_main_t * vm, adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX]; adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX]; - /* We should never rewrite a pkt using the MISS adjacency */ - ASSERT (adj_index0 && adj_index1); - ip0 = vlib_buffer_get_current (p0); ip1 = vlib_buffer_get_current (p1); error0 = error1 = IP6_ERROR_NONE; next0 = next1 = IP6_REWRITE_NEXT_DROP; - if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_LOCALLY_ORIGINATED))) + is_locally_originated0 = + p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED; + if (PREDICT_TRUE (!is_locally_originated0)) { i32 hop_limit0 = ip0->hop_limit; @@ -2069,9 +1688,11 @@ ip6_rewrite_inline (vlib_main_t * vm, } else { - p0->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED; + p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED; } - if (PREDICT_TRUE (!(p1->flags & VNET_BUFFER_LOCALLY_ORIGINATED))) + is_locally_originated1 = + p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED; + if (PREDICT_TRUE (!is_locally_originated1)) { i32 hop_limit1 = ip1->hop_limit; @@ -2098,39 +1719,51 @@ ip6_rewrite_inline (vlib_main_t * vm, } else { - p1->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED; + p1->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED; } - adj0 = ip_get_adjacency (lm, adj_index0); - adj1 = ip_get_adjacency (lm, adj_index1); + adj0 = adj_get (adj_index0); + adj1 = adj_get (adj_index1); rw_len0 = adj0[0].rewrite_header.data_bytes; rw_len1 = adj1[0].rewrite_header.data_bytes; vnet_buffer (p0)->ip.save_rewrite_length = rw_len0; vnet_buffer (p1)->ip.save_rewrite_length = rw_len1; - vlib_increment_combined_counter - (&adjacency_counters, - cpu_index, - adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); - vlib_increment_combined_counter - (&adjacency_counters, - cpu_index, adj_index1, - 1, vlib_buffer_length_in_chain (vm, p1) + rw_len1); + if (do_counters) + { + vlib_increment_combined_counter + (&adjacency_counters, + thread_index, adj_index0, 1, + vlib_buffer_length_in_chain (vm, p0) + rw_len0); + vlib_increment_combined_counter + (&adjacency_counters, + thread_index, adj_index1, 1, + vlib_buffer_length_in_chain (vm, p1) + rw_len1); + } /* Check MTU of outgoing interface. */ - error0 = - (vlib_buffer_length_in_chain (vm, p0) > - adj0[0]. - rewrite_header.max_l3_packet_bytes ? IP6_ERROR_MTU_EXCEEDED : - error0); - error1 = - (vlib_buffer_length_in_chain (vm, p1) > - adj1[0]. - rewrite_header.max_l3_packet_bytes ? IP6_ERROR_MTU_EXCEEDED : - error1); + u16 ip0_len = + clib_net_to_host_u16 (ip0->payload_length) + + sizeof (ip6_header_t); + u16 ip1_len = + clib_net_to_host_u16 (ip1->payload_length) + + sizeof (ip6_header_t); + if (do_gso && (p0->flags & VNET_BUFFER_F_GSO)) + ip0_len = gso_mtu_sz (p0); + if (do_gso && (p1->flags & VNET_BUFFER_F_GSO)) + ip1_len = gso_mtu_sz (p1); + + + + ip6_mtu_check (p0, ip0_len, + adj0[0].rewrite_header.max_l3_packet_bytes, + is_locally_originated0, &next0, &error0); + ip6_mtu_check (p1, ip1_len, + adj1[0].rewrite_header.max_l3_packet_bytes, + is_locally_originated1, &next1, &error1); /* Don't adjust the buffer for hop count issue; icmp-error node - * wants to see the IP headerr */ + * wants to see the IP header */ if (PREDICT_TRUE (error0 == IP6_ERROR_NONE)) { p0->current_data -= rw_len0; @@ -2140,8 +1773,14 @@ ip6_rewrite_inline (vlib_main_t * vm, vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0; next0 = adj0[0].rewrite_header.next_index; - vnet_feature_arc_start (lm->output_feature_arc_index, - tx_sw_if_index0, &next0, p0); + if (PREDICT_FALSE + (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES)) + vnet_feature_arc_start (lm->output_feature_arc_index, + tx_sw_if_index0, &next0, p0); + } + else + { + p0->error = error_node->errors[error0]; } if (PREDICT_TRUE (error1 == IP6_ERROR_NONE)) { @@ -2152,8 +1791,22 @@ ip6_rewrite_inline (vlib_main_t * vm, vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1; next1 = adj1[0].rewrite_header.next_index; - vnet_feature_arc_start (lm->output_feature_arc_index, - tx_sw_if_index1, &next1, p1); + if (PREDICT_FALSE + (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES)) + vnet_feature_arc_start (lm->output_feature_arc_index, + tx_sw_if_index1, &next1, p1); + } + else + { + p1->error = error_node->errors[error1]; + } + + if (is_midchain) + { + /* before we paint on the next header, update the L4 + * checksums if required, since there's no offload on a tunnel */ + calc_checksums (vm, p0); + calc_checksums (vm, p1); } /* Guess we are only writing on simple Ethernet header. */ @@ -2162,8 +1815,28 @@ ip6_rewrite_inline (vlib_main_t * vm, if (is_midchain) { - adj0->sub_type.midchain.fixup_func (vm, adj0, p0); - adj1->sub_type.midchain.fixup_func (vm, adj1, p1); + if (adj0->sub_type.midchain.fixup_func) + adj0->sub_type.midchain.fixup_func + (vm, adj0, p0, adj0->sub_type.midchain.fixup_data); + if (adj1->sub_type.midchain.fixup_func) + adj1->sub_type.midchain.fixup_func + (vm, adj1, p1, adj1->sub_type.midchain.fixup_data); + } + if (is_mcast) + { + /* + * copy bytes from the IP address into the MAC rewrite + */ + vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK, + adj0-> + rewrite_header.dst_mcast_offset, + &ip0->dst_address.as_u32[3], + (u8 *) ip0); + vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK, + adj1-> + rewrite_header.dst_mcast_offset, + &ip1->dst_address.as_u32[3], + (u8 *) ip1); } vlib_validate_buffer_enqueue_x2 (vm, node, next_index, @@ -2179,6 +1852,7 @@ ip6_rewrite_inline (vlib_main_t * vm, u32 pi0, rw_len0; u32 adj_index0, next0, error0; u32 tx_sw_if_index0; + bool is_locally_originated0; pi0 = to_next[0] = from[0]; @@ -2186,10 +1860,7 @@ ip6_rewrite_inline (vlib_main_t * vm, adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX]; - /* We should never rewrite a pkt using the MISS adjacency */ - ASSERT (adj_index0); - - adj0 = ip_get_adjacency (lm, adj_index0); + adj0 = adj_get (adj_index0); ip0 = vlib_buffer_get_current (p0); @@ -2197,7 +1868,9 @@ ip6_rewrite_inline (vlib_main_t * vm, next0 = IP6_REWRITE_NEXT_DROP; /* Check hop limit */ - if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_LOCALLY_ORIGINATED))) + is_locally_originated0 = + p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED; + if (PREDICT_TRUE (!is_locally_originated0)) { i32 hop_limit0 = ip0->hop_limit; @@ -2223,7 +1896,12 @@ ip6_rewrite_inline (vlib_main_t * vm, } else { - p0->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED; + p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED; + } + + if (is_midchain) + { + calc_checksums (vm, p0); } /* Guess we are only writing on simple Ethernet header. */ @@ -2233,20 +1911,27 @@ ip6_rewrite_inline (vlib_main_t * vm, rw_len0 = adj0[0].rewrite_header.data_bytes; vnet_buffer (p0)->ip.save_rewrite_length = rw_len0; - vlib_increment_combined_counter - (&adjacency_counters, - cpu_index, - adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); + if (do_counters) + { + vlib_increment_combined_counter + (&adjacency_counters, + thread_index, adj_index0, 1, + vlib_buffer_length_in_chain (vm, p0) + rw_len0); + } /* Check MTU of outgoing interface. */ - error0 = - (vlib_buffer_length_in_chain (vm, p0) > - adj0[0]. - rewrite_header.max_l3_packet_bytes ? IP6_ERROR_MTU_EXCEEDED : - error0); + u16 ip0_len = + clib_net_to_host_u16 (ip0->payload_length) + + sizeof (ip6_header_t); + if (do_gso && (p0->flags & VNET_BUFFER_F_GSO)) + ip0_len = gso_mtu_sz (p0); + + ip6_mtu_check (p0, ip0_len, + adj0[0].rewrite_header.max_l3_packet_bytes, + is_locally_originated0, &next0, &error0); /* Don't adjust the buffer for hop count issue; icmp-error node - * wants to see the IP headerr */ + * wants to see the IP header */ if (PREDICT_TRUE (error0 == IP6_ERROR_NONE)) { p0->current_data -= rw_len0; @@ -2257,16 +1942,30 @@ ip6_rewrite_inline (vlib_main_t * vm, vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0; next0 = adj0[0].rewrite_header.next_index; - vnet_feature_arc_start (lm->output_feature_arc_index, - tx_sw_if_index0, &next0, p0); + if (PREDICT_FALSE + (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES)) + vnet_feature_arc_start (lm->output_feature_arc_index, + tx_sw_if_index0, &next0, p0); + } + else + { + p0->error = error_node->errors[error0]; } if (is_midchain) { - adj0->sub_type.midchain.fixup_func (vm, adj0, p0); + if (adj0->sub_type.midchain.fixup_func) + adj0->sub_type.midchain.fixup_func + (vm, adj0, p0, adj0->sub_type.midchain.fixup_data); + } + if (is_mcast) + { + vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK, + adj0-> + rewrite_header.dst_mcast_offset, + &ip0->dst_address.as_u32[3], + (u8 *) ip0); } - - p0->error = error_node->errors[error0]; from += 1; n_left_from -= 1; @@ -2288,71 +1987,144 @@ ip6_rewrite_inline (vlib_main_t * vm, return frame->n_vectors; } -static uword -ip6_rewrite (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) +always_inline uword +ip6_rewrite_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame, + int do_counters, int is_midchain, int is_mcast) +{ + vnet_main_t *vnm = vnet_get_main (); + if (PREDICT_FALSE (vnm->interface_main.gso_interface_count > 0)) + return ip6_rewrite_inline_with_gso (vm, node, frame, do_counters, + is_midchain, is_mcast, + 1 /* do_gso */ ); + else + return ip6_rewrite_inline_with_gso (vm, node, frame, do_counters, + is_midchain, is_mcast, + 0 /* no do_gso */ ); +} + +VLIB_NODE_FN (ip6_rewrite_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + if (adj_are_counters_enabled ()) + return ip6_rewrite_inline (vm, node, frame, 1, 0, 0); + else + return ip6_rewrite_inline (vm, node, frame, 0, 0, 0); +} + +VLIB_NODE_FN (ip6_rewrite_bcast_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + if (adj_are_counters_enabled ()) + return ip6_rewrite_inline (vm, node, frame, 1, 0, 0); + else + return ip6_rewrite_inline (vm, node, frame, 0, 0, 0); +} + +VLIB_NODE_FN (ip6_rewrite_mcast_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + if (adj_are_counters_enabled ()) + return ip6_rewrite_inline (vm, node, frame, 1, 0, 1); + else + return ip6_rewrite_inline (vm, node, frame, 0, 0, 1); +} + +VLIB_NODE_FN (ip6_midchain_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { - return ip6_rewrite_inline (vm, node, frame, - /* midchain */ 0); + if (adj_are_counters_enabled ()) + return ip6_rewrite_inline (vm, node, frame, 1, 1, 0); + else + return ip6_rewrite_inline (vm, node, frame, 0, 1, 0); } -static uword -ip6_midchain (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) +VLIB_NODE_FN (ip6_mcast_midchain_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { - return ip6_rewrite_inline (vm, node, frame, - /* midchain */ 1); + if (adj_are_counters_enabled ()) + return ip6_rewrite_inline (vm, node, frame, 1, 1, 1); + else + return ip6_rewrite_inline (vm, node, frame, 0, 1, 1); } /* *INDENT-OFF* */ VLIB_REGISTER_NODE (ip6_midchain_node) = { - .function = ip6_midchain, .name = "ip6-midchain", .vector_size = sizeof (u32), .format_trace = format_ip6_forward_next_trace, .sibling_of = "ip6-rewrite", }; -/* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (ip6_midchain_node, ip6_midchain); - -/* *INDENT-OFF* */ VLIB_REGISTER_NODE (ip6_rewrite_node) = { - .function = ip6_rewrite, .name = "ip6-rewrite", .vector_size = sizeof (u32), .format_trace = format_ip6_rewrite_trace, - .n_next_nodes = 2, + .n_next_nodes = IP6_REWRITE_N_NEXT, .next_nodes = { - [IP6_REWRITE_NEXT_DROP] = "error-drop", + [IP6_REWRITE_NEXT_DROP] = "ip6-drop", [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error", + [IP6_REWRITE_NEXT_FRAGMENT] = "ip6-frag", }, }; -/* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite); +VLIB_REGISTER_NODE (ip6_rewrite_bcast_node) = { + .name = "ip6-rewrite-bcast", + .vector_size = sizeof (u32), + + .format_trace = format_ip6_rewrite_trace, + .sibling_of = "ip6-rewrite", +}; + +VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) = +{ + .name = "ip6-rewrite-mcast", + .vector_size = sizeof (u32), + .format_trace = format_ip6_rewrite_trace, + .sibling_of = "ip6-rewrite", +}; + + +VLIB_REGISTER_NODE (ip6_mcast_midchain_node) = +{ + .name = "ip6-mcast-midchain", + .vector_size = sizeof (u32), + .format_trace = format_ip6_rewrite_trace, + .sibling_of = "ip6-rewrite", +}; + +/* *INDENT-ON* */ /* * Hop-by-Hop handling */ - +#ifndef CLIB_MARCH_VARIANT ip6_hop_by_hop_main_t ip6_hop_by_hop_main; +#endif /* CLIB_MARCH_VARIANT */ #define foreach_ip6_hop_by_hop_error \ _(PROCESSED, "pkts with ip6 hop-by-hop options") \ _(FORMAT, "incorrectly formatted hop-by-hop options") \ _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options") +/* *INDENT-OFF* */ typedef enum { #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym, foreach_ip6_hop_by_hop_error #undef _ - IP6_HOP_BY_HOP_N_ERROR, + IP6_HOP_BY_HOP_N_ERROR, } ip6_hop_by_hop_error_t; +/* *INDENT-ON* */ /* * Primary h-b-h handler trace support @@ -2365,7 +2137,7 @@ typedef struct u8 option_data[256]; } ip6_hop_by_hop_trace_t; -vlib_node_registration_t ip6_hop_by_hop_node; +extern vlib_node_registration_t ip6_hop_by_hop_node; static char *ip6_hop_by_hop_error_strings[] = { #define _(sym,string) string, @@ -2373,6 +2145,52 @@ static char *ip6_hop_by_hop_error_strings[] = { #undef _ }; +#ifndef CLIB_MARCH_VARIANT +u8 * +format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args) +{ + ip6_hop_by_hop_header_t *hbh0 = va_arg (*args, ip6_hop_by_hop_header_t *); + int total_len = va_arg (*args, int); + ip6_hop_by_hop_option_t *opt0, *limit0; + ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main; + u8 type0; + + s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d", + hbh0->protocol, (hbh0->length + 1) << 3, total_len); + + opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1); + limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 + total_len); + + while (opt0 < limit0) + { + type0 = opt0->type; + switch (type0) + { + case 0: /* Pad, just stop */ + opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0 + 1); + break; + + default: + if (hm->trace[type0]) + { + s = (*hm->trace[type0]) (s, opt0); + } + else + { + s = + format (s, "\n unrecognized option %d length %d", type0, + opt0->length); + } + opt0 = + (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length + + sizeof (ip6_hop_by_hop_option_t)); + break; + } + } + return s; +} +#endif + static u8 * format_ip6_hop_by_hop_trace (u8 * s, va_list * args) { @@ -2499,17 +2317,15 @@ ip6_scan_hbh_options (vlib_buffer_t * b0, /* * Process the Hop-by-Hop Options header */ -static uword -ip6_hop_by_hop (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * frame) +VLIB_NODE_FN (ip6_hop_by_hop_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { vlib_node_runtime_t *error_node = vlib_node_get_runtime (vm, ip6_hop_by_hop_node.index); ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main; u32 n_left_from, *from, *to_next; ip_lookup_next_t next_index; - ip6_main_t *im = &ip6_main; - ip_lookup_main_t *lm = &im->lookup_main; from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -2558,9 +2374,9 @@ ip6_hop_by_hop (vlib_main_t * vm, /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */ u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX]; - ip_adjacency_t *adj0 = ip_get_adjacency (lm, adj_index0); + ip_adjacency_t *adj0 = adj_get (adj_index0); u32 adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX]; - ip_adjacency_t *adj1 = ip_get_adjacency (lm, adj_index1); + ip_adjacency_t *adj1 = adj_get (adj_index1); /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */ next0 = adj0->lookup_next_index; @@ -2629,7 +2445,7 @@ ip6_hop_by_hop (vlib_main_t * vm, ARRAY_LEN (t->option_data) ? trace_len : ARRAY_LEN (t->option_data); t->trace_len = trace_len; - clib_memcpy (t->option_data, hbh0, trace_len); + clib_memcpy_fast (t->option_data, hbh0, trace_len); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -2643,7 +2459,7 @@ ip6_hop_by_hop (vlib_main_t * vm, ARRAY_LEN (t->option_data) ? trace_len : ARRAY_LEN (t->option_data); t->trace_len = trace_len; - clib_memcpy (t->option_data, hbh1, trace_len); + clib_memcpy_fast (t->option_data, hbh1, trace_len); } } @@ -2681,7 +2497,7 @@ ip6_hop_by_hop (vlib_main_t * vm, * A HBH option rarely redirects to a different node */ u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX]; - ip_adjacency_t *adj0 = ip_get_adjacency (lm, adj_index0); + ip_adjacency_t *adj0 = adj_get (adj_index0); next0 = adj0->lookup_next_index; ip0 = vlib_buffer_get_current (b0); @@ -2724,7 +2540,7 @@ ip6_hop_by_hop (vlib_main_t * vm, ARRAY_LEN (t->option_data) ? trace_len : ARRAY_LEN (t->option_data); t->trace_len = trace_len; - clib_memcpy (t->option_data, hbh0, trace_len); + clib_memcpy_fast (t->option_data, hbh0, trace_len); } b0->error = error_node->errors[error0]; @@ -2741,7 +2557,6 @@ ip6_hop_by_hop (vlib_main_t * vm, /* *INDENT-OFF* */ VLIB_REGISTER_NODE (ip6_hop_by_hop_node) = { - .function = ip6_hop_by_hop, .name = "ip6-hop-by-hop", .sibling_of = "ip6-lookup", .vector_size = sizeof (u32), @@ -2753,20 +2568,19 @@ VLIB_REGISTER_NODE (ip6_hop_by_hop_node) = }; /* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (ip6_hop_by_hop_node, ip6_hop_by_hop); - static clib_error_t * ip6_hop_by_hop_init (vlib_main_t * vm) { ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main; - memset (hm->options, 0, sizeof (hm->options)); - memset (hm->trace, 0, sizeof (hm->trace)); + clib_memset (hm->options, 0, sizeof (hm->options)); + clib_memset (hm->trace, 0, sizeof (hm->trace)); hm->next_override = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP; return (0); } VLIB_INIT_FUNCTION (ip6_hop_by_hop_init); +#ifndef CLIB_MARCH_VARIANT void ip6_hbh_set_next_override (uword next) { @@ -2784,7 +2598,7 @@ ip6_hbh_register_option (u8 option, ip6_main_t *im = &ip6_main; ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main; - ASSERT (option < ARRAY_LEN (hm->options)); + ASSERT ((u32) option < ARRAY_LEN (hm->options)); /* Already registered */ if (hm->options[option]) @@ -2805,7 +2619,7 @@ ip6_hbh_unregister_option (u8 option) ip6_main_t *im = &ip6_main; ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main; - ASSERT (option < ARRAY_LEN (hm->options)); + ASSERT ((u32) option < ARRAY_LEN (hm->options)); /* Not registered */ if (!hm->options[option]) @@ -2833,6 +2647,7 @@ ip6_hbh_unregister_option (u8 option) /* Global IP6 main. */ ip6_main_t ip6_main; +#endif static clib_error_t * ip6_lookup_init (vlib_main_t * vm) @@ -2869,15 +2684,21 @@ ip6_lookup_init (vlib_main_t * vm) if (im->lookup_table_size == 0) im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE; - BV (clib_bihash_init) (&(im->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash), + clib_bihash_init_24_8 (&(im->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash), "ip6 FIB fwding table", im->lookup_table_nbuckets, im->lookup_table_size); - BV (clib_bihash_init) (&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash, + clib_bihash_init_24_8 (&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash, "ip6 FIB non-fwding table", im->lookup_table_nbuckets, im->lookup_table_size); + clib_bihash_init_40_8 (&im->ip6_mtable.ip6_mhash, + "ip6 mFIB table", + im->lookup_table_nbuckets, im->lookup_table_size); /* Create FIB with index 0 and table id of 0. */ - fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0); + fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0, + FIB_SOURCE_DEFAULT_ROUTE); + mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0, + MFIB_SOURCE_DEFAULT_ROUTE); { pg_node_t *pn; @@ -2891,7 +2712,7 @@ ip6_lookup_init (vlib_main_t * vm) { icmp6_neighbor_solicitation_header_t p; - memset (&p, 0, sizeof (p)); + clib_memset (&p, 0, sizeof (p)); p.ip.ip_version_traffic_class_and_flow_label = clib_host_to_net_u32 (0x6 << 28); @@ -2922,104 +2743,6 @@ ip6_lookup_init (vlib_main_t * vm) VLIB_INIT_FUNCTION (ip6_lookup_init); -static clib_error_t * -add_del_ip6_interface_table (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - vnet_main_t *vnm = vnet_get_main (); - clib_error_t *error = 0; - u32 sw_if_index, table_id; - - sw_if_index = ~0; - - if (!unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index)) - { - error = clib_error_return (0, "unknown interface `%U'", - format_unformat_error, input); - goto done; - } - - if (unformat (input, "%d", &table_id)) - ; - else - { - error = clib_error_return (0, "expected table id `%U'", - format_unformat_error, input); - goto done; - } - - { - u32 fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, - table_id); - - vec_validate (ip6_main.fib_index_by_sw_if_index, sw_if_index); - ip6_main.fib_index_by_sw_if_index[sw_if_index] = fib_index; - } - - -done: - return error; -} - -/*? - * Place the indicated interface into the supplied IPv6 FIB table (also known - * as a VRF). If the FIB table does not exist, this command creates it. To - * display the current IPv6 FIB table, use the command 'show ip6 fib'. - * FIB table will only be displayed if a route has been added to the table, or - * an IP Address is assigned to an interface in the table (which adds a route - * automatically). - * - * @note IP addresses added after setting the interface IP table end up in - * the indicated FIB table. If the IP address is added prior to adding the - * interface to the FIB table, it will NOT be part of the FIB table. Predictable - * but potentially counter-intuitive results occur if you provision interface - * addresses in multiple FIBs. Upon RX, packets will be processed in the last - * IP table ID provisioned. It might be marginally useful to evade source RPF - * drops to put an interface address into multiple FIBs. - * - * @cliexpar - * Example of how to add an interface to an IPv6 FIB table (where 2 is the table-id): - * @cliexcmd{set interface ip6 table GigabitEthernet2/0/0 2} - ?*/ -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (set_interface_ip6_table_command, static) = -{ - .path = "set interface ip6 table", - .function = add_del_ip6_interface_table, - .short_help = "set interface ip6 table " -}; -/* *INDENT-ON* */ - -void -ip6_link_local_address_from_ethernet_mac_address (ip6_address_t * ip, - u8 * mac) -{ - ip->as_u64[0] = clib_host_to_net_u64 (0xFE80000000000000ULL); - /* Invert the "u" bit */ - ip->as_u8[8] = mac[0] ^ (1 << 1); - ip->as_u8[9] = mac[1]; - ip->as_u8[10] = mac[2]; - ip->as_u8[11] = 0xFF; - ip->as_u8[12] = 0xFE; - ip->as_u8[13] = mac[3]; - ip->as_u8[14] = mac[4]; - ip->as_u8[15] = mac[5]; -} - -void -ip6_ethernet_mac_address_from_link_local_address (u8 * mac, - ip6_address_t * ip) -{ - /* Invert the previously inverted "u" bit */ - mac[0] = ip->as_u8[8] ^ (1 << 1); - mac[1] = ip->as_u8[9]; - mac[2] = ip->as_u8[10]; - mac[3] = ip->as_u8[13]; - mac[4] = ip->as_u8[14]; - mac[5] = ip->as_u8[15]; -} - static clib_error_t * test_ip6_link_command_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) @@ -3059,21 +2782,23 @@ VLIB_CLI_COMMAND (test_link_command, static) = }; /* *INDENT-ON* */ +#ifndef CLIB_MARCH_VARIANT int vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config) { - ip6_main_t *im6 = &ip6_main; - ip6_fib_t *fib; - uword *p = hash_get (im6->fib_index_by_table_id, table_id); + u32 fib_index; + + fib_index = fib_table_find (FIB_PROTOCOL_IP6, table_id); - if (p == 0) - return -1; + if (~0 == fib_index) + return VNET_API_ERROR_NO_SUCH_FIB; - fib = ip6_fib_get (p[0]); + fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP6, + flow_hash_config); - fib->flow_hash_config = flow_hash_config; - return 1; + return 0; } +#endif static clib_error_t * set_ip6_flow_hash_command_fn (vlib_main_t * vm, @@ -3104,7 +2829,7 @@ set_ip6_flow_hash_command_fn (vlib_main_t * vm, rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config); switch (rv) { - case 1: + case 0: break; case -1: @@ -3215,7 +2940,14 @@ show_ip6_local_command_fn (vlib_main_t * vm, for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++) { if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT) - vlib_cli_output (vm, "%d", i); + { + + u32 node_index = vlib_get_node (vm, + ip6_local_node.index)-> + next_nodes[lm->local_next_by_ip_protocol[i]]; + vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm, + node_index); + } } return 0; } @@ -3244,6 +2976,7 @@ VLIB_CLI_COMMAND (show_ip6_local, static) = }; /* *INDENT-ON* */ +#ifndef CLIB_MARCH_VARIANT int vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index, u32 table_index) @@ -3264,7 +2997,7 @@ vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index, vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index); lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index; - if_addr = ip6_interface_first_address (ipm, sw_if_index, NULL); + if_addr = ip6_interface_first_address (ipm, sw_if_index); if (NULL != if_addr) { @@ -3303,6 +3036,7 @@ vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index, return 0; } +#endif static clib_error_t * set_ip6_classify_command_fn (vlib_main_t * vm, @@ -3379,14 +3113,9 @@ ip6_config (vlib_main_t * vm, unformat_input_t * input) { if (unformat (input, "hash-buckets %d", &tmp)) nbuckets = tmp; - else if (unformat (input, "heap-size %dm", &tmp)) - heapsize = ((u64) tmp) << 20; - else if (unformat (input, "heap-size %dM", &tmp)) - heapsize = ((u64) tmp) << 20; - else if (unformat (input, "heap-size %dg", &tmp)) - heapsize = ((u64) tmp) << 30; - else if (unformat (input, "heap-size %dG", &tmp)) - heapsize = ((u64) tmp) << 30; + else if (unformat (input, "heap-size %U", + unformat_memory_size, &heapsize)) + ; else return clib_error_return (0, "unknown input '%U'", format_unformat_error, input);