X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fadj%2Fadj_midchain.c;h=3d879e9d7fc51bc38f5310965a23913cd7f6e3f3;hb=8f5fef2c7;hp=9fd3246b15a441d8db6a53ebf53e48ad9ed96e84;hpb=db14f5aff69311d13a0a90baa3567e92f3faa783;p=vpp.git diff --git a/src/vnet/adj/adj_midchain.c b/src/vnet/adj/adj_midchain.c index 9fd3246b15a..3d879e9d7fc 100644 --- a/src/vnet/adj/adj_midchain.c +++ b/src/vnet/adj/adj_midchain.c @@ -20,13 +20,11 @@ #include #include #include +#include #include - -/** - * The two midchain tx feature node indices - */ -static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM]; -static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM]; +#include +#include +#include /** * @brief Trace data for packets traversing the midchain tx node @@ -45,212 +43,158 @@ adj_midchain_tx_inline (vlib_main_t * vm, vlib_frame_t * frame, int interface_count) { - u32 * from, * to_next, n_left_from, n_left_to_next; - u32 next_index; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; + u16 nexts[VLIB_FRAME_SIZE], *next; + u32 * from, n_left, thread_index; vnet_main_t *vnm = vnet_get_main (); vnet_interface_main_t *im = &vnm->interface_main; - u32 thread_index = vm->thread_index; - /* Vector of buffer / pkt indices we're supposed to process */ + thread_index = vm->thread_index; + n_left = frame->n_vectors; from = vlib_frame_vector_args (frame); - /* Number of buffers / pkts */ - n_left_from = frame->n_vectors; + vlib_get_buffers (vm, from, bufs, n_left); - /* Speculatively send the first buffer to the last disposition we used */ - next_index = node->cached_next_index; + next = nexts; + b = bufs; - while (n_left_from > 0) + while (n_left > 8) { - /* set up to enqueue to our disposition with index = next_index */ - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + u32 adj_index0, adj_index1, adj_index2, adj_index3; + const ip_adjacency_t *adj0, *adj1, *adj2, *adj3; + const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3; - while (n_left_from >= 8 && n_left_to_next > 4) - { - u32 bi0, adj_index0, next0; - const ip_adjacency_t * adj0; - const dpo_id_t *dpo0; - vlib_buffer_t * b0; - u32 bi1, adj_index1, next1; - const ip_adjacency_t * adj1; - const dpo_id_t *dpo1; - vlib_buffer_t * b1; - u32 bi2, adj_index2, next2; - const ip_adjacency_t * adj2; - const dpo_id_t *dpo2; - vlib_buffer_t * b2; - u32 bi3, adj_index3, next3; - const ip_adjacency_t * adj3; - const dpo_id_t *dpo3; - vlib_buffer_t * b3; - - /* Prefetch next iteration. */ - { - vlib_buffer_t * p4, * p5; - vlib_buffer_t * p6, * p7; - - p4 = vlib_get_buffer (vm, from[4]); - p5 = vlib_get_buffer (vm, from[5]); - p6 = vlib_get_buffer (vm, from[6]); - p7 = vlib_get_buffer (vm, from[7]); - - vlib_prefetch_buffer_header (p4, LOAD); - vlib_prefetch_buffer_header (p5, LOAD); - vlib_prefetch_buffer_header (p6, LOAD); - vlib_prefetch_buffer_header (p7, LOAD); - } - - bi0 = from[0]; - to_next[0] = bi0; - bi1 = from[1]; - to_next[1] = bi1; - bi2 = from[2]; - to_next[2] = bi2; - bi3 = from[3]; - to_next[3] = bi3; - - from += 4; - to_next += 4; - n_left_from -= 4; - n_left_to_next -= 4; - - b0 = vlib_get_buffer(vm, bi0); - b1 = vlib_get_buffer(vm, bi1); - b2 = vlib_get_buffer(vm, bi2); - b3 = vlib_get_buffer(vm, bi3); - - /* Follow the DPO on which the midchain is stacked */ - adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX]; - adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX]; - adj_index2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX]; - adj_index3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX]; - - adj0 = adj_get(adj_index0); - adj1 = adj_get(adj_index1); - adj2 = adj_get(adj_index2); - adj3 = adj_get(adj_index3); - - dpo0 = &adj0->sub_type.midchain.next_dpo; - dpo1 = &adj1->sub_type.midchain.next_dpo; - dpo2 = &adj2->sub_type.midchain.next_dpo; - dpo3 = &adj3->sub_type.midchain.next_dpo; - - next0 = dpo0->dpoi_next_node; - next1 = dpo1->dpoi_next_node; - next2 = dpo2->dpoi_next_node; - next3 = dpo3->dpoi_next_node; - - vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; - vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - vnet_buffer(b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index; - vnet_buffer(b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index; - - if (interface_count) - { - vlib_increment_combined_counter (im->combined_sw_if_counters - + VNET_INTERFACE_COUNTER_TX, - thread_index, - adj0->rewrite_header.sw_if_index, - 1, - vlib_buffer_length_in_chain (vm, b0)); - vlib_increment_combined_counter (im->combined_sw_if_counters - + VNET_INTERFACE_COUNTER_TX, - thread_index, - adj1->rewrite_header.sw_if_index, - 1, - vlib_buffer_length_in_chain (vm, b1)); - vlib_increment_combined_counter (im->combined_sw_if_counters - + VNET_INTERFACE_COUNTER_TX, - thread_index, - adj2->rewrite_header.sw_if_index, - 1, - vlib_buffer_length_in_chain (vm, b2)); - vlib_increment_combined_counter (im->combined_sw_if_counters - + VNET_INTERFACE_COUNTER_TX, - thread_index, - adj3->rewrite_header.sw_if_index, - 1, - vlib_buffer_length_in_chain (vm, b3)); - } - - if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) - { - adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node, - b0, sizeof (*tr)); - tr->ai = adj_index0; - } - if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED)) - { - adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node, - b1, sizeof (*tr)); - tr->ai = adj_index1; - } - if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED)) - { - adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node, - b2, sizeof (*tr)); - tr->ai = adj_index2; - } - if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED)) - { - adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node, - b3, sizeof (*tr)); - tr->ai = adj_index3; - } - - vlib_validate_buffer_enqueue_x4 (vm, node, next_index, - to_next, n_left_to_next, - bi0, bi1, bi2, bi3, - next0, next1, next2, next3); - } - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0, adj_index0, next0; - const ip_adjacency_t * adj0; - const dpo_id_t *dpo0; - vlib_buffer_t * b0; - - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer(vm, bi0); - - /* Follow the DPO on which the midchain is stacked */ - adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX]; - adj0 = adj_get(adj_index0); - dpo0 = &adj0->sub_type.midchain.next_dpo; - next0 = dpo0->dpoi_next_node; - vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - - if (interface_count) - { - vlib_increment_combined_counter (im->combined_sw_if_counters - + VNET_INTERFACE_COUNTER_TX, - thread_index, - adj0->rewrite_header.sw_if_index, - 1, - vlib_buffer_length_in_chain (vm, b0)); - } - - if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) - { - adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node, - b0, sizeof (*tr)); - tr->ai = adj_index0; - } - - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, next0); - } + /* Prefetch next iteration. */ + { + vlib_prefetch_buffer_header (b[4], LOAD); + vlib_prefetch_buffer_header (b[5], LOAD); + vlib_prefetch_buffer_header (b[6], LOAD); + vlib_prefetch_buffer_header (b[7], LOAD); + } + + /* Follow the DPO on which the midchain is stacked */ + adj_index0 = vnet_buffer(b[0])->ip.adj_index[VLIB_TX]; + adj_index1 = vnet_buffer(b[1])->ip.adj_index[VLIB_TX]; + adj_index2 = vnet_buffer(b[2])->ip.adj_index[VLIB_TX]; + adj_index3 = vnet_buffer(b[3])->ip.adj_index[VLIB_TX]; + + adj0 = adj_get(adj_index0); + adj1 = adj_get(adj_index1); + adj2 = adj_get(adj_index2); + adj3 = adj_get(adj_index3); + + dpo0 = &adj0->sub_type.midchain.next_dpo; + dpo1 = &adj1->sub_type.midchain.next_dpo; + dpo2 = &adj2->sub_type.midchain.next_dpo; + dpo3 = &adj3->sub_type.midchain.next_dpo; + + next[0] = dpo0->dpoi_next_node; + next[1] = dpo1->dpoi_next_node; + next[2] = dpo2->dpoi_next_node; + next[3] = dpo3->dpoi_next_node; + + vnet_buffer(b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; + vnet_buffer(b[1])->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; + vnet_buffer(b[2])->ip.adj_index[VLIB_TX] = dpo2->dpoi_index; + vnet_buffer(b[3])->ip.adj_index[VLIB_TX] = dpo3->dpoi_index; + + if (interface_count) + { + vlib_increment_combined_counter (im->combined_sw_if_counters + + VNET_INTERFACE_COUNTER_TX, + thread_index, + adj0->rewrite_header.sw_if_index, + 1, + vlib_buffer_length_in_chain (vm, b[0])); + vlib_increment_combined_counter (im->combined_sw_if_counters + + VNET_INTERFACE_COUNTER_TX, + thread_index, + adj1->rewrite_header.sw_if_index, + 1, + vlib_buffer_length_in_chain (vm, b[1])); + vlib_increment_combined_counter (im->combined_sw_if_counters + + VNET_INTERFACE_COUNTER_TX, + thread_index, + adj2->rewrite_header.sw_if_index, + 1, + vlib_buffer_length_in_chain (vm, b[2])); + vlib_increment_combined_counter (im->combined_sw_if_counters + + VNET_INTERFACE_COUNTER_TX, + thread_index, + adj3->rewrite_header.sw_if_index, + 1, + vlib_buffer_length_in_chain (vm, b[3])); + } + + if (PREDICT_FALSE(node->flags & VLIB_NODE_FLAG_TRACE)) + { + if (PREDICT_FALSE(b[0]->flags & VLIB_BUFFER_IS_TRACED)) + { + adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node, + b[0], sizeof (*tr)); + tr->ai = adj_index0; + } + if (PREDICT_FALSE(b[1]->flags & VLIB_BUFFER_IS_TRACED)) + { + adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node, + b[1], sizeof (*tr)); + tr->ai = adj_index1; + } + if (PREDICT_FALSE(b[2]->flags & VLIB_BUFFER_IS_TRACED)) + { + adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node, + b[2], sizeof (*tr)); + tr->ai = adj_index2; + } + if (PREDICT_FALSE(b[3]->flags & VLIB_BUFFER_IS_TRACED)) + { + adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node, + b[3], sizeof (*tr)); + tr->ai = adj_index3; + } + } + n_left -= 4; + b += 4; + next += 4; + } + + while (n_left) + { + const ip_adjacency_t * adj0; + const dpo_id_t *dpo0; + u32 adj_index0; + + /* Follow the DPO on which the midchain is stacked */ + adj_index0 = vnet_buffer(b[0])->ip.adj_index[VLIB_TX]; + adj0 = adj_get(adj_index0); + dpo0 = &adj0->sub_type.midchain.next_dpo; + next[0] = dpo0->dpoi_next_node; + vnet_buffer(b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; + + if (interface_count) + { + vlib_increment_combined_counter (im->combined_sw_if_counters + + VNET_INTERFACE_COUNTER_TX, + thread_index, + adj0->rewrite_header.sw_if_index, + 1, + vlib_buffer_length_in_chain (vm, b[0])); + } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); + if (PREDICT_FALSE(b[0]->flags & VLIB_BUFFER_IS_TRACED)) + { + adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node, + b[0], sizeof (*tr)); + tr->ai = adj_index0; + } + + n_left -= 1; + b += 1; + next += 1; } + vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); + return frame->n_vectors; } @@ -276,7 +220,7 @@ adj_midchain_tx (vlib_main_t * vm, return (adj_midchain_tx_inline(vm, node, frame, 1)); } -VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = { +VLIB_REGISTER_NODE (adj_midchain_tx_node) = { .function = adj_midchain_tx, .name = "adj-midchain-tx", .vector_size = sizeof (u32), @@ -297,79 +241,44 @@ adj_midchain_tx_no_count (vlib_main_t * vm, return (adj_midchain_tx_inline(vm, node, frame, 0)); } -VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = { +VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node) = { .function = adj_midchain_tx_no_count, .name = "adj-midchain-tx-no-count", .vector_size = sizeof (u32), .format_trace = format_adj_midchain_tx_trace, - - .n_next_nodes = 1, - .next_nodes = { - [0] = "error-drop", - }, + .sibling_of = "adj-midchain-tx", }; -VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = { - .arc_name = "ip4-output", - .node_name = "adj-midchain-tx", - .runs_before = VNET_FEATURES ("interface-output"), - .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4], -}; -VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = { - .arc_name = "ip4-output", - .node_name = "adj-midchain-tx-no-count", - .runs_before = VNET_FEATURES ("interface-output"), - .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4], -}; -VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = { - .arc_name = "ip6-output", - .node_name = "adj-midchain-tx", - .runs_before = VNET_FEATURES ("interface-output"), - .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6], -}; -VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = { - .arc_name = "ip6-output", - .node_name = "adj-midchain-tx-no-count", - .runs_before = VNET_FEATURES ("interface-output"), - .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6], -}; -VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = { - .arc_name = "mpls-output", - .node_name = "adj-midchain-tx", - .runs_before = VNET_FEATURES ("interface-output"), - .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS], -}; -VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = { - .arc_name = "mpls-output", - .node_name = "adj-midchain-tx-no-count", - .runs_before = VNET_FEATURES ("interface-output"), - .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS], -}; -VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = { - .arc_name = "ethernet-output", - .node_name = "adj-midchain-tx", - .runs_before = VNET_FEATURES ("error-drop"), - .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET], -}; -VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = { - .arc_name = "ethernet-output", - .node_name = "adj-midchain-tx-no-count", - .runs_before = VNET_FEATURES ("error-drop"), - .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET], -}; -VNET_FEATURE_INIT (adj_midchain_tx_nsh, static) = { - .arc_name = "nsh-output", - .node_name = "adj-midchain-tx", - .runs_before = VNET_FEATURES ("error-drop"), - .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_NSH], -}; -VNET_FEATURE_INIT (adj_midchain_tx_no_count_nsh, static) = { - .arc_name = "nsh-output", - .node_name = "adj-midchain-tx-no-count", - .runs_before = VNET_FEATURES ("error-drop"), - .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_NSH], -}; +#ifndef CLIB_MARCH_VARIANT + +u8 +adj_is_midchain (adj_index_t ai) +{ + ip_adjacency_t *adj; + + adj = adj_get(ai); + + switch (adj->lookup_next_index) + { + case IP_LOOKUP_NEXT_MIDCHAIN: + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: + return (1); + case IP_LOOKUP_NEXT_ARP: + case IP_LOOKUP_NEXT_GLEAN: + case IP_LOOKUP_NEXT_BCAST: + case IP_LOOKUP_NEXT_MCAST: + case IP_LOOKUP_NEXT_DROP: + case IP_LOOKUP_NEXT_PUNT: + case IP_LOOKUP_NEXT_LOCAL: + case IP_LOOKUP_NEXT_REWRITE: + case IP_LOOKUP_NEXT_ICMP_ERROR: + case IP_LOOKUP_N_NEXT: + return (0); + } + + return (0); +} static inline u32 adj_get_midchain_node (vnet_link_t link) @@ -420,7 +329,7 @@ adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj) } case VNET_LINK_NSH: { - arc = nsh_main_dummy.output_feature_arc_index; + arc = nsh_main_placeholder.output_feature_arc_index; break; } case VNET_LINK_ARP: @@ -441,15 +350,25 @@ adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj) adj_midchain_tx_node.index); } -static u32 -adj_nbr_midchain_get_feature_node (ip_adjacency_t *adj) +/** + * adj_midchain_setup + * + * Setup the adj as a mid-chain + */ +void +adj_midchain_teardown (ip_adjacency_t *adj) { - if (adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) - { - return (adj_midchain_tx_no_count_feature_node[adj->ia_link]); - } + vlib_main_t *vm = vlib_get_main(); - return (adj_midchain_tx_feature_node[adj->ia_link]); + dpo_reset(&adj->sub_type.midchain.next_dpo); + + vlib_worker_thread_barrier_sync(vm); + adj->ia_cfg_index = vnet_feature_modify_end_node( + adj_midchain_get_feature_arc_index_for_link_type (adj), + adj->rewrite_header.sw_if_index, + vlib_get_node_by_name (vlib_get_main(), + (u8*) "interface-output")->index); + vlib_worker_thread_barrier_release(vm); } /** @@ -463,9 +382,9 @@ adj_midchain_setup (adj_index_t adj_index, const void *data, adj_flags_t flags) { - u32 feature_index, tx_node; + vlib_main_t *vm = vlib_get_main(); ip_adjacency_t *adj; - u8 arc_index; + u32 tx_node; ASSERT(ADJ_INDEX_INVALID != adj_index); @@ -473,15 +392,30 @@ adj_midchain_setup (adj_index_t adj_index, adj->sub_type.midchain.fixup_func = fixup; adj->sub_type.midchain.fixup_data = data; + adj->sub_type.midchain.fei = FIB_NODE_INDEX_INVALID; adj->ia_flags |= flags; - arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj); - feature_index = adj_nbr_midchain_get_feature_node(adj); + if (flags & ADJ_FLAG_MIDCHAIN_FIXUP_IP4O4_HDR) + { + adj->rewrite_header.flags |= VNET_REWRITE_FIXUP_IP4_O_4; + } + else + { + adj->rewrite_header.flags &= ~VNET_REWRITE_FIXUP_IP4_O_4; + } + if (!(flags & ADJ_FLAG_MIDCHAIN_FIXUP_FLOW_HASH)) + { + adj->rewrite_header.flags &= ~VNET_REWRITE_FIXUP_FLOW_HASH; + } + tx_node = adj_nbr_midchain_get_tx_node(adj); - vnet_feature_enable_disable_with_index (arc_index, feature_index, - adj->rewrite_header.sw_if_index, - 1 /* enable */, 0, 0); + vlib_worker_thread_barrier_sync(vm); + adj->ia_cfg_index = vnet_feature_modify_end_node( + adj_midchain_get_feature_arc_index_for_link_type (adj), + adj->rewrite_header.sw_if_index, + tx_node); + vlib_worker_thread_barrier_release(vm); /* * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx. @@ -499,7 +433,7 @@ adj_midchain_setup (adj_index_t adj_index, * adj_nbr_midchain_update_rewrite * * Update the adjacency's rewrite string. A NULL string implies the - * rewrite is reset (i.e. when ARP/ND etnry is gone). + * rewrite is reset (i.e. when ARP/ND entry is gone). * NB: the adj being updated may be handling traffic in the DP. */ void @@ -516,19 +450,17 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index, adj = adj_get(adj_index); /* - * one time only update. since we don't support chainging the tunnel + * one time only update. since we don't support changing the tunnel * src,dst, this is all we need. */ - ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP); - /* - * tunnels can always provide a rewrite. - */ - ASSERT(NULL != rewrite); - - adj_midchain_setup(adj_index, fixup, fixup_data, flags); + if (adj->lookup_next_index != IP_LOOKUP_NEXT_MIDCHAIN && + adj->lookup_next_index != IP_LOOKUP_NEXT_MCAST_MIDCHAIN) + { + adj_midchain_setup(adj_index, fixup, fixup_data, flags); + } /* - * update the rewirte with the workers paused. + * update the rewrite with the workers paused. */ adj_nbr_update_rewrite_internal(adj, IP_LOOKUP_NEXT_MIDCHAIN, @@ -537,6 +469,58 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index, rewrite); } +void +adj_nbr_midchain_update_next_node (adj_index_t adj_index, + u32 next_node) +{ + ip_adjacency_t *adj; + vlib_main_t * vm; + + ASSERT(ADJ_INDEX_INVALID != adj_index); + + adj = adj_get(adj_index); + vm = vlib_get_main(); + + vlib_worker_thread_barrier_sync(vm); + + adj->rewrite_header.next_index = vlib_node_add_next(vlib_get_main(), + adj->ia_node_index, + next_node); + + adj->ia_cfg_index = vnet_feature_modify_end_node( + adj_midchain_get_feature_arc_index_for_link_type (adj), + adj->rewrite_header.sw_if_index, + next_node); + + vlib_worker_thread_barrier_release(vm); +} + +void +adj_nbr_midchain_reset_next_node (adj_index_t adj_index) +{ + ip_adjacency_t *adj; + vlib_main_t * vm; + + ASSERT(ADJ_INDEX_INVALID != adj_index); + + adj = adj_get(adj_index); + vm = vlib_get_main(); + + vlib_worker_thread_barrier_sync(vm); + + adj->rewrite_header.next_index = + vlib_node_add_next(vlib_get_main(), + adj->ia_node_index, + adj_nbr_midchain_get_tx_node(adj)); + + adj->ia_cfg_index = vnet_feature_modify_end_node( + adj_midchain_get_feature_arc_index_for_link_type (adj), + adj->rewrite_header.sw_if_index, + adj_nbr_midchain_get_tx_node(adj)); + + vlib_worker_thread_barrier_release(vm); +} + /** * adj_nbr_midchain_unstack * @@ -545,11 +529,24 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index, void adj_nbr_midchain_unstack (adj_index_t adj_index) { + fib_node_index_t *entry_indicies, tmp; ip_adjacency_t *adj; ASSERT(ADJ_INDEX_INVALID != adj_index); + adj = adj_get (adj_index); - adj = adj_get(adj_index); + /* + * check to see if this unstacking breaks a recursion loop + */ + entry_indicies = NULL; + tmp = adj->sub_type.midchain.fei; + adj->sub_type.midchain.fei = FIB_NODE_INDEX_INVALID; + + if (FIB_NODE_INDEX_INVALID != tmp) + { + fib_entry_recursive_loop_detect(tmp, &entry_indicies); + vec_free(entry_indicies); + } /* * stack on the drop @@ -561,6 +558,97 @@ adj_nbr_midchain_unstack (adj_index_t adj_index) CLIB_MEMORY_BARRIER(); } +void +adj_nbr_midchain_stack_on_fib_entry (adj_index_t ai, + fib_node_index_t fei, + fib_forward_chain_type_t fct) +{ + fib_node_index_t *entry_indicies; + dpo_id_t tmp = DPO_INVALID; + ip_adjacency_t *adj; + + adj = adj_get (ai); + + /* + * check to see if this stacking will form a recursion loop + */ + entry_indicies = NULL; + adj->sub_type.midchain.fei = fei; + + if (fib_entry_recursive_loop_detect(adj->sub_type.midchain.fei, &entry_indicies)) + { + /* + * loop formed, stack on the drop. + */ + dpo_copy(&tmp, drop_dpo_get(fib_forw_chain_type_to_dpo_proto(fct))); + } + else + { + fib_entry_contribute_forwarding (fei, fct, &tmp); + + if (DPO_LOAD_BALANCE == tmp.dpoi_type) + { + load_balance_t *lb; + + lb = load_balance_get (tmp.dpoi_index); + + if ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_IP_STACK) || + lb->lb_n_buckets == 1) + { + /* + * do that hash now and stack on the choice. + * If the choice is an incomplete adj then we will need a poke when + * it becomes complete. This happens since the adj update walk propagates + * as far a recursive paths. + */ + const dpo_id_t *choice; + int hash; + + if (FIB_FORW_CHAIN_TYPE_UNICAST_IP4 == fct) + { + hash = ip4_compute_flow_hash ((ip4_header_t *) adj_get_rewrite (ai), + lb->lb_hash_config); + } + else if (FIB_FORW_CHAIN_TYPE_UNICAST_IP6 == fct) + { + hash = ip6_compute_flow_hash ((ip6_header_t *) adj_get_rewrite (ai), + lb->lb_hash_config); + } + else + { + hash = 0; + ASSERT(0); + } + + choice = load_balance_get_bucket_i (lb, hash & lb->lb_n_buckets_minus_1); + dpo_copy (&tmp, choice); + } + else if (lb->lb_n_buckets > 1) + { + /* + * the client has chosen not to use the stacking to select a + * bucket, and there are more than one buckets. there's no + * value in using the midchain's fixed rewrite string to select + * the path, so force a flow hash on the inner. + */ + adj->rewrite_header.flags |= VNET_REWRITE_FIXUP_FLOW_HASH; + } + + if (adj->ia_flags & ADJ_FLAG_MIDCHAIN_FIXUP_FLOW_HASH) + { + /* + * The client, for reasons unbeknownst to adj, wants to force + * a flow hash on the inner, we will oblige. + */ + adj->rewrite_header.flags |= VNET_REWRITE_FIXUP_FLOW_HASH; + } + } + } + adj_nbr_midchain_stack (ai, &tmp); + dpo_reset(&tmp); + vec_free(entry_indicies); +} + /** * adj_nbr_midchain_stack */ @@ -582,6 +670,33 @@ adj_nbr_midchain_stack (adj_index_t adj_index, next); } +int +adj_ndr_midchain_recursive_loop_detect (adj_index_t ai, + fib_node_index_t **entry_indicies) +{ + fib_node_index_t *entry_index, *entries; + ip_adjacency_t * adj; + + adj = adj_get(ai); + entries = *entry_indicies; + + vec_foreach(entry_index, entries) + { + if (*entry_index == adj->sub_type.midchain.fei) + { + /* + * The entry this midchain links to is already in the set + * of visited entries, this is a loop + */ + adj->ia_flags |= ADJ_FLAG_MIDCHAIN_LOOPED; + return (1); + } + } + + adj->ia_flags &= ~ADJ_FLAG_MIDCHAIN_LOOPED; + return (0); +} + u8* format_adj_midchain (u8* s, va_list *ap) { @@ -590,13 +705,23 @@ format_adj_midchain (u8* s, va_list *ap) ip_adjacency_t * adj = adj_get(index); s = format (s, "%U", format_vnet_link, adj->ia_link); - s = format (s, " via %U ", - format_ip46_address, &adj->sub_type.nbr.next_hop); + if (adj->rewrite_header.flags & VNET_REWRITE_HAS_FEATURES) + s = format(s, " [features]"); + s = format (s, " via %U", + format_ip46_address, &adj->sub_type.nbr.next_hop, + adj_proto_to_46(adj->ia_nh_proto)); s = format (s, " %U", format_vnet_rewrite, &adj->rewrite_header, sizeof (adj->rewrite_data), indent); - s = format (s, "\n%Ustacked-on:\n%U%U", - format_white_space, indent, + s = format (s, "\n%Ustacked-on", + format_white_space, indent); + + if (FIB_NODE_INDEX_INVALID != adj->sub_type.midchain.fei) + { + s = format (s, " entry:%d", adj->sub_type.midchain.fei); + + } + s = format (s, ":\n%U%U", format_white_space, indent+2, format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2); @@ -619,6 +744,7 @@ const static dpo_vft_t adj_midchain_dpo_vft = { .dv_unlock = adj_dpo_unlock, .dv_format = format_adj_midchain, .dv_get_urpf = adj_dpo_get_urpf, + .dv_get_mtu = adj_dpo_get_mtu, }; /** @@ -668,3 +794,5 @@ adj_midchain_module_init (void) { dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes); } + +#endif