X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fip%2Fip4_forward.c;h=43213fe8ace6d58b5bd23269529406f5e212d750;hb=b538dd868;hp=1a1b7e5182b08783d9acace04578ab88dd445f57;hpb=45db885a6a050aa809ae333bf6c14349db644822;p=vpp.git diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c index 1a1b7e5182b..43213fe8ace 100644 --- a/src/vnet/ip/ip4_forward.c +++ b/src/vnet/ip/ip4_forward.c @@ -55,6 +55,7 @@ #include /* for mFIB table and entry creation */ #include +#include /** @brief IPv4 lookup node. @node ip4-lookup @@ -115,182 +116,149 @@ VLIB_NODE_FN (ip4_load_balance_node) (vlib_main_t * vm, vlib_frame_t * frame) { vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters; - u32 n_left_from, n_left_to_next, *from, *to_next; - ip_lookup_next_t next; + u32 n_left, *from; u32 thread_index = vm->thread_index; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; + u16 nexts[VLIB_FRAME_SIZE], *next; from = vlib_frame_vector_args (frame); - n_left_from = frame->n_vectors; - next = node->cached_next_index; - - while (n_left_from > 0) - { - vlib_get_next_frame (vm, node, next, to_next, n_left_to_next); - - - while (n_left_from >= 4 && n_left_to_next >= 2) - { - ip_lookup_next_t next0, next1; - const load_balance_t *lb0, *lb1; - vlib_buffer_t *p0, *p1; - u32 pi0, lbi0, hc0, pi1, lbi1, hc1; - const ip4_header_t *ip0, *ip1; - const dpo_id_t *dpo0, *dpo1; - - /* Prefetch next iteration. */ - { - vlib_buffer_t *p2, *p3; - - p2 = vlib_get_buffer (vm, from[2]); - p3 = vlib_get_buffer (vm, from[3]); - - vlib_prefetch_buffer_header (p2, STORE); - vlib_prefetch_buffer_header (p3, STORE); + n_left = frame->n_vectors; + next = nexts; - CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE); - CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE); - } + vlib_get_buffers (vm, from, bufs, n_left); - pi0 = to_next[0] = from[0]; - pi1 = to_next[1] = from[1]; + while (n_left >= 4) + { + const load_balance_t *lb0, *lb1; + const ip4_header_t *ip0, *ip1; + u32 lbi0, hc0, lbi1, hc1; + const dpo_id_t *dpo0, *dpo1; - from += 2; - n_left_from -= 2; - to_next += 2; - n_left_to_next -= 2; + /* Prefetch next iteration. */ + { + vlib_prefetch_buffer_header (b[2], LOAD); + vlib_prefetch_buffer_header (b[3], LOAD); - p0 = vlib_get_buffer (vm, pi0); - p1 = vlib_get_buffer (vm, pi1); + CLIB_PREFETCH (b[2]->data, sizeof (ip0[0]), LOAD); + CLIB_PREFETCH (b[3]->data, sizeof (ip0[0]), LOAD); + } - ip0 = vlib_buffer_get_current (p0); - ip1 = vlib_buffer_get_current (p1); - lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX]; - lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX]; + ip0 = vlib_buffer_get_current (b[0]); + ip1 = vlib_buffer_get_current (b[1]); + lbi0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX]; + lbi1 = vnet_buffer (b[1])->ip.adj_index[VLIB_TX]; - lb0 = load_balance_get (lbi0); - lb1 = load_balance_get (lbi1); + lb0 = load_balance_get (lbi0); + lb1 = load_balance_get (lbi1); - /* - * this node is for via FIBs we can re-use the hash value from the - * to node if present. - * We don't want to use the same hash value at each level in the recursion - * graph as that would lead to polarisation - */ - hc0 = hc1 = 0; + /* + * this node is for via FIBs we can re-use the hash value from the + * to node if present. + * We don't want to use the same hash value at each level in the recursion + * graph as that would lead to polarisation + */ + hc0 = hc1 = 0; - if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) + if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) + { + if (PREDICT_TRUE (vnet_buffer (b[0])->ip.flow_hash)) { - if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash)) - { - hc0 = vnet_buffer (p0)->ip.flow_hash = - vnet_buffer (p0)->ip.flow_hash >> 1; - } - else - { - hc0 = vnet_buffer (p0)->ip.flow_hash = - ip4_compute_flow_hash (ip0, lb0->lb_hash_config); - } - dpo0 = load_balance_get_fwd_bucket - (lb0, (hc0 & (lb0->lb_n_buckets_minus_1))); + hc0 = vnet_buffer (b[0])->ip.flow_hash = + vnet_buffer (b[0])->ip.flow_hash >> 1; } else { - dpo0 = load_balance_get_bucket_i (lb0, 0); + hc0 = vnet_buffer (b[0])->ip.flow_hash = + ip4_compute_flow_hash (ip0, lb0->lb_hash_config); } - if (PREDICT_FALSE (lb1->lb_n_buckets > 1)) + dpo0 = load_balance_get_fwd_bucket + (lb0, (hc0 & (lb0->lb_n_buckets_minus_1))); + } + else + { + dpo0 = load_balance_get_bucket_i (lb0, 0); + } + if (PREDICT_FALSE (lb1->lb_n_buckets > 1)) + { + if (PREDICT_TRUE (vnet_buffer (b[1])->ip.flow_hash)) { - if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash)) - { - hc1 = vnet_buffer (p1)->ip.flow_hash = - vnet_buffer (p1)->ip.flow_hash >> 1; - } - else - { - hc1 = vnet_buffer (p1)->ip.flow_hash = - ip4_compute_flow_hash (ip1, lb1->lb_hash_config); - } - dpo1 = load_balance_get_fwd_bucket - (lb1, (hc1 & (lb1->lb_n_buckets_minus_1))); + hc1 = vnet_buffer (b[1])->ip.flow_hash = + vnet_buffer (b[1])->ip.flow_hash >> 1; } else { - dpo1 = load_balance_get_bucket_i (lb1, 0); + hc1 = vnet_buffer (b[1])->ip.flow_hash = + ip4_compute_flow_hash (ip1, lb1->lb_hash_config); } + dpo1 = load_balance_get_fwd_bucket + (lb1, (hc1 & (lb1->lb_n_buckets_minus_1))); + } + else + { + dpo1 = load_balance_get_bucket_i (lb1, 0); + } - next0 = dpo0->dpoi_next_node; - next1 = dpo1->dpoi_next_node; - - vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; - - vlib_increment_combined_counter - (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); - vlib_increment_combined_counter - (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); + next[0] = dpo0->dpoi_next_node; + next[1] = dpo1->dpoi_next_node; - vlib_validate_buffer_enqueue_x2 (vm, node, next, - to_next, n_left_to_next, - pi0, pi1, next0, next1); - } + vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; + vnet_buffer (b[1])->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; - while (n_left_from > 0 && n_left_to_next > 0) - { - ip_lookup_next_t next0; - const load_balance_t *lb0; - vlib_buffer_t *p0; - u32 pi0, lbi0, hc0; - const ip4_header_t *ip0; - const dpo_id_t *dpo0; + vlib_increment_combined_counter + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b[0])); + vlib_increment_combined_counter + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, b[1])); - pi0 = from[0]; - to_next[0] = pi0; - from += 1; - to_next += 1; - n_left_to_next -= 1; - n_left_from -= 1; + b += 2; + next += 2; + n_left -= 2; + } - p0 = vlib_get_buffer (vm, pi0); + while (n_left > 0) + { + const load_balance_t *lb0; + const ip4_header_t *ip0; + const dpo_id_t *dpo0; + u32 lbi0, hc0; - ip0 = vlib_buffer_get_current (p0); - lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX]; + ip0 = vlib_buffer_get_current (b[0]); + lbi0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX]; - lb0 = load_balance_get (lbi0); + lb0 = load_balance_get (lbi0); - hc0 = 0; - if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) + hc0 = 0; + if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) + { + if (PREDICT_TRUE (vnet_buffer (b[0])->ip.flow_hash)) { - if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash)) - { - hc0 = vnet_buffer (p0)->ip.flow_hash = - vnet_buffer (p0)->ip.flow_hash >> 1; - } - else - { - hc0 = vnet_buffer (p0)->ip.flow_hash = - ip4_compute_flow_hash (ip0, lb0->lb_hash_config); - } - dpo0 = load_balance_get_fwd_bucket - (lb0, (hc0 & (lb0->lb_n_buckets_minus_1))); + hc0 = vnet_buffer (b[0])->ip.flow_hash = + vnet_buffer (b[0])->ip.flow_hash >> 1; } else { - dpo0 = load_balance_get_bucket_i (lb0, 0); + hc0 = vnet_buffer (b[0])->ip.flow_hash = + ip4_compute_flow_hash (ip0, lb0->lb_hash_config); } + dpo0 = load_balance_get_fwd_bucket + (lb0, (hc0 & (lb0->lb_n_buckets_minus_1))); + } + else + { + dpo0 = load_balance_get_bucket_i (lb0, 0); + } - next0 = dpo0->dpoi_next_node; - vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - - vlib_increment_combined_counter - (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + next[0] = dpo0->dpoi_next_node; + vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - vlib_validate_buffer_enqueue_x1 (vm, node, next, - to_next, n_left_to_next, - pi0, next0); - } + vlib_increment_combined_counter + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b[0])); - vlib_put_next_frame (vm, node, next, n_left_to_next); + b += 1; + next += 1; + n_left -= 1; } + vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); if (node->flags & VLIB_NODE_FLAG_TRACE) ip4_forward_next_trace (vm, node, frame, VLIB_TX); @@ -547,6 +515,12 @@ ip4_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable) vnet_feature_enable_disable ("ip4-multicast", "ip4-not-enabled", sw_if_index, !is_enable, 0, 0); + + { + ip4_enable_disable_interface_callback_t *cb; + vec_foreach (cb, im->enable_disable_interface_callbacks) + cb->function (im, cb->function_opaque, sw_if_index, is_enable); + } } static clib_error_t * @@ -900,7 +874,9 @@ ip4_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add) VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del); /* Global IP4 main. */ +#ifndef CLIB_MARCH_VARIANT ip4_main_t ip4_main; +#endif /* CLIB_MARCH_VARIANT */ static clib_error_t * ip4_lookup_init (vlib_main_t * vm) @@ -948,10 +924,6 @@ ip4_lookup_init (vlib_main_t * vm) clib_memset (&h, 0, sizeof (h)); - /* Set target ethernet address to all zeros. */ - clib_memset (h.ip4_over_ethernet[1].ethernet, 0, - sizeof (h.ip4_over_ethernet[1].ethernet)); - #define _16(f,v) h.f = clib_host_to_net_u16 (v); #define _8(f,v) h.f = v; _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet); @@ -1175,7 +1147,7 @@ ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT); p0 = vlib_get_buffer (vm, p0->next_buffer); data_this_buffer = vlib_buffer_get_current (p0); - n_this_buffer = p0->current_length; + n_this_buffer = clib_min (p0->current_length, n_bytes_left); } sum16 = ~ip_csum_fold (sum0); @@ -1707,6 +1679,16 @@ ip4_register_protocol (u32 protocol, u32 node_index) lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip4_local_node.index, node_index); } + +void +ip4_unregister_protocol (u32 protocol) +{ + ip4_main_t *im = &ip4_main; + ip_lookup_main_t *lm = &im->lookup_main; + + ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol)); + lm->local_next_by_ip_protocol[protocol] = IP_LOCAL_NEXT_PUNT; +} #endif static clib_error_t * @@ -1725,8 +1707,8 @@ show_ip_local_command_fn (vlib_main_t * vm, u32 node_index = vlib_get_node (vm, ip4_local_node.index)-> next_nodes[lm->local_next_by_ip_protocol[i]]; - vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm, - node_index); + vlib_cli_output (vm, "%U: %U", format_ip_protocol, i, + format_vlib_node_name, vm, node_index); } } return 0; @@ -1854,11 +1836,6 @@ ip4_arp_inline (vlib_main_t * vm, vlib_packet_template_get_packet (vm, &im->ip4_arp_request_packet_template, &bi0); - b0 = vlib_get_buffer (vm, bi0); - - /* copy the persistent fields from the original */ - clib_memcpy_fast (b0->opaque2, p0->opaque2, sizeof (p0->opaque2)); - /* Seems we're out of buffers */ if (PREDICT_FALSE (!h0)) { @@ -1866,15 +1843,19 @@ ip4_arp_inline (vlib_main_t * vm, continue; } + b0 = vlib_get_buffer (vm, bi0); + + /* copy the persistent fields from the original */ + clib_memcpy_fast (b0->opaque2, p0->opaque2, sizeof (p0->opaque2)); + /* Add rewrite/encap string for ARP packet. */ vnet_rewrite_one_header (adj0[0], h0, sizeof (ethernet_header_t)); hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); /* Src ethernet address in ARP header. */ - clib_memcpy_fast (h0->ip4_over_ethernet[0].ethernet, - hw_if0->hw_address, - sizeof (h0->ip4_over_ethernet[0].ethernet)); + mac_address_from_bytes (&h0->ip4_over_ethernet[0].mac, + hw_if0->hw_address); if (is_glean) { /* The interface's source address is stashed in the Glean Adj */ @@ -2046,8 +2027,7 @@ ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index, sw_if_index); } - clib_memcpy_fast (h->ip4_over_ethernet[0].ethernet, hi->hw_address, - sizeof (h->ip4_over_ethernet[0].ethernet)); + mac_address_from_bytes (&h->ip4_over_ethernet[0].mac, hi->hw_address); h->ip4_over_ethernet[0].ip4 = src[0]; h->ip4_over_ethernet[1].ip4 = dst[0]; @@ -2190,10 +2170,11 @@ ip4_ttl_and_checksum_check (vlib_buffer_t * b, ip4_header_t * ip, u16 * next, always_inline uword -ip4_rewrite_inline (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame, - int do_counters, int is_midchain, int is_mcast) +ip4_rewrite_inline_with_gso (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame, + int do_counters, int is_midchain, int is_mcast, + int do_gso) { ip_lookup_main_t *lm = &ip4_main.lookup_main; u32 *from = vlib_frame_vector_args (frame); @@ -2271,12 +2252,20 @@ ip4_rewrite_inline (vlib_main_t * vm, CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); /* Check MTU of outgoing interface. */ - ip4_mtu_check (b[0], clib_net_to_host_u16 (ip0->length), + u16 ip0_len = clib_net_to_host_u16 (ip0->length); + u16 ip1_len = clib_net_to_host_u16 (ip1->length); + + if (do_gso && (b[0]->flags & VNET_BUFFER_F_GSO)) + ip0_len = gso_mtu_sz (b[0]); + if (do_gso && (b[1]->flags & VNET_BUFFER_F_GSO)) + ip1_len = gso_mtu_sz (b[1]); + + ip4_mtu_check (b[0], ip0_len, adj0[0].rewrite_header.max_l3_packet_bytes, ip0->flags_and_fragment_offset & clib_host_to_net_u16 (IP4_HEADER_FLAG_DONT_FRAGMENT), next + 0, &error0); - ip4_mtu_check (b[1], clib_net_to_host_u16 (ip1->length), + ip4_mtu_check (b[1], ip1_len, adj1[0].rewrite_header.max_l3_packet_bytes, ip1->flags_and_fragment_offset & clib_host_to_net_u16 (IP4_HEADER_FLAG_DONT_FRAGMENT), @@ -2292,10 +2281,8 @@ ip4_rewrite_inline (vlib_main_t * vm, IP4_ERROR_SAME_INTERFACE : error1); } - b[0]->error = error_node->errors[error0]; - b[1]->error = error_node->errors[error1]; /* Don't adjust the buffer for ttl issue; icmp-error node wants - * to see the IP headerr */ + * to see the IP header */ if (PREDICT_TRUE (error0 == IP4_ERROR_NONE)) { u32 next_index = adj0[0].rewrite_header.next_index; @@ -2310,6 +2297,10 @@ ip4_rewrite_inline (vlib_main_t * vm, tx_sw_if_index0, &next_index, b[0]); next[0] = next_index; } + else + { + b[0]->error = error_node->errors[error0]; + } if (PREDICT_TRUE (error1 == IP4_ERROR_NONE)) { u32 next_index = adj1[0].rewrite_header.next_index; @@ -2325,7 +2316,15 @@ ip4_rewrite_inline (vlib_main_t * vm, tx_sw_if_index1, &next_index, b[1]); next[1] = next_index; } - + else + { + b[1]->error = error_node->errors[error1]; + } + if (is_midchain) + { + calc_checksums (vm, b[0]); + calc_checksums (vm, b[1]); + } /* Guess we are only writing on simple Ethernet header. */ vnet_rewrite_two_headers (adj0[0], adj1[0], ip0, ip1, sizeof (ethernet_header_t)); @@ -2348,10 +2347,12 @@ ip4_rewrite_inline (vlib_main_t * vm, if (is_midchain) { - adj0->sub_type.midchain.fixup_func - (vm, adj0, b[0], adj0->sub_type.midchain.fixup_data); - adj1->sub_type.midchain.fixup_func - (vm, adj1, b[1], adj1->sub_type.midchain.fixup_data); + if (adj0->sub_type.midchain.fixup_func) + adj0->sub_type.midchain.fixup_func + (vm, adj0, b[0], adj0->sub_type.midchain.fixup_data); + if (adj1->sub_type.midchain.fixup_func) + adj1->sub_type.midchain.fixup_func + (vm, adj1, b[1], adj1->sub_type.midchain.fixup_data); } if (is_mcast) @@ -2399,7 +2400,11 @@ ip4_rewrite_inline (vlib_main_t * vm, vnet_buffer (b[0])->ip.save_rewrite_length = rw_len0; /* Check MTU of outgoing interface. */ - ip4_mtu_check (b[0], clib_net_to_host_u16 (ip0->length), + u16 ip0_len = clib_net_to_host_u16 (ip0->length); + if (do_gso && (b[0]->flags & VNET_BUFFER_F_GSO)) + ip0_len = gso_mtu_sz (b[0]); + + ip4_mtu_check (b[0], ip0_len, adj0[0].rewrite_header.max_l3_packet_bytes, ip0->flags_and_fragment_offset & clib_host_to_net_u16 (IP4_HEADER_FLAG_DONT_FRAGMENT), @@ -2411,10 +2416,9 @@ ip4_rewrite_inline (vlib_main_t * vm, vnet_buffer (b[0])->sw_if_index[VLIB_RX]) ? IP4_ERROR_SAME_INTERFACE : error0); } - b[0]->error = error_node->errors[error0]; /* Don't adjust the buffer for ttl issue; icmp-error node wants - * to see the IP headerr */ + * to see the IP header */ if (PREDICT_TRUE (error0 == IP4_ERROR_NONE)) { u32 next_index = adj0[0].rewrite_header.next_index; @@ -2429,7 +2433,14 @@ ip4_rewrite_inline (vlib_main_t * vm, tx_sw_if_index0, &next_index, b[0]); next[0] = next_index; } - + else + { + b[0]->error = error_node->errors[error0]; + } + if (is_midchain) + { + calc_checksums (vm, b[0]); + } /* Guess we are only writing on simple Ethernet header. */ vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t)); @@ -2441,8 +2452,9 @@ ip4_rewrite_inline (vlib_main_t * vm, if (is_midchain) { - adj0->sub_type.midchain.fixup_func - (vm, adj0, b[0], adj0->sub_type.midchain.fixup_data); + if (adj0->sub_type.midchain.fixup_func) + adj0->sub_type.midchain.fixup_func + (vm, adj0, b[0], adj0->sub_type.midchain.fixup_data); } if (is_mcast) @@ -2469,6 +2481,23 @@ ip4_rewrite_inline (vlib_main_t * vm, return frame->n_vectors; } +always_inline uword +ip4_rewrite_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame, + int do_counters, int is_midchain, int is_mcast) +{ + vnet_main_t *vnm = vnet_get_main (); + if (PREDICT_FALSE (vnm->interface_main.gso_interface_count > 0)) + return ip4_rewrite_inline_with_gso (vm, node, frame, do_counters, + is_midchain, is_mcast, + 1 /* do_gso */ ); + else + return ip4_rewrite_inline_with_gso (vm, node, frame, do_counters, + is_midchain, is_mcast, + 0 /* no do_gso */ ); +} + /** @brief IPv4 rewrite node. @node ip4-rewrite