src/vnet/adj/adj_midchain.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/adj/adj_nbr.h>
  17 #include <vnet/adj/adj_internal.h>
  18 #include <vnet/adj/adj_l2.h>
  19 #include <vnet/adj/adj_nsh.h>
  20 #include <vnet/adj/adj_midchain.h>
  21 #include <vnet/ethernet/arp_packet.h>
  22 #include <vnet/dpo/drop_dpo.h>
  23 #include <vnet/dpo/load_balance.h>
  24 #include <vnet/fib/fib_walk.h>
  25 #include <vnet/fib/fib_entry.h>
  26
  27 /**
  28  * The two midchain tx feature node indices
  29  */
  30 static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM];
  31 static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM];
  32
  33 /**
  34  * @brief Trace data for packets traversing the midchain tx node
  35  */
  36 typedef struct adj_midchain_tx_trace_t_
  37 {
  38     /**
  39      * @brief the midchain adj we are traversing
  40      */
  41     adj_index_t ai;
  42 } adj_midchain_tx_trace_t;
  43
  44 always_inline uword
  45 adj_midchain_tx_inline (vlib_main_t * vm,
  46                         vlib_node_runtime_t * node,
  47                         vlib_frame_t * frame,
  48                         int interface_count)
  49 {
  50     u32 * from, * to_next, n_left_from, n_left_to_next;
  51     u32 next_index;
  52     vnet_main_t *vnm = vnet_get_main ();
  53     vnet_interface_main_t *im = &vnm->interface_main;
  54     u32 thread_index = vm->thread_index;
  55
  56     /* Vector of buffer / pkt indices we're supposed to process */
  57     from = vlib_frame_vector_args (frame);
  58
  59     /* Number of buffers / pkts */
  60     n_left_from = frame->n_vectors;
  61
  62     /* Speculatively send the first buffer to the last disposition we used */
  63     next_index = node->cached_next_index;
  64
  65     while (n_left_from > 0)
  66     {
  67         /* set up to enqueue to our disposition with index = next_index */
  68         vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
  69
  70         while (n_left_from >= 8 && n_left_to_next > 4)
  71         {
  72             const ip_adjacency_t *adj0, *adj1, *adj2, *adj3;
  73             const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3;
  74             vlib_buffer_t * b0, *b1, *b2, *b3;
  75             u32 bi0, adj_index0, next0;
  76             u32 bi1, adj_index1, next1;
  77             u32 bi2, adj_index2, next2;
  78             u32 bi3, adj_index3, next3;
  79
  80             /* Prefetch next iteration. */
  81             {
  82                 vlib_buffer_t * p4, * p5;
  83                 vlib_buffer_t * p6, * p7;
  84
  85                 p4 = vlib_get_buffer (vm, from[4]);
  86                 p5 = vlib_get_buffer (vm, from[5]);
  87                 p6 = vlib_get_buffer (vm, from[6]);
  88                 p7 = vlib_get_buffer (vm, from[7]);
  89
  90                 vlib_prefetch_buffer_header (p4, LOAD);
  91                 vlib_prefetch_buffer_header (p5, LOAD);
  92                 vlib_prefetch_buffer_header (p6, LOAD);
  93                 vlib_prefetch_buffer_header (p7, LOAD);
  94             }
  95
  96             bi0 = from[0];
  97             to_next[0] = bi0;
  98             bi1 = from[1];
  99             to_next[1] = bi1;
 100             bi2 = from[2];
 101             to_next[2] = bi2;
 102             bi3 = from[3];
 103             to_next[3] = bi3;
 104
 105             from += 4;
 106             to_next += 4;
 107             n_left_from -= 4;
 108             n_left_to_next -= 4;
 109
 110             b0 = vlib_get_buffer(vm, bi0);
 111             b1 = vlib_get_buffer(vm, bi1);
 112             b2 = vlib_get_buffer(vm, bi2);
 113             b3 = vlib_get_buffer(vm, bi3);
 114
 115             /* Follow the DPO on which the midchain is stacked */
 116             adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 117             adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 118             adj_index2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
 119             adj_index3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
 120
 121             adj0 = adj_get(adj_index0);
 122             adj1 = adj_get(adj_index1);
 123             adj2 = adj_get(adj_index2);
 124             adj3 = adj_get(adj_index3);
 125
 126             dpo0 = &adj0->sub_type.midchain.next_dpo;
 127             dpo1 = &adj1->sub_type.midchain.next_dpo;
 128             dpo2 = &adj2->sub_type.midchain.next_dpo;
 129             dpo3 = &adj3->sub_type.midchain.next_dpo;
 130
 131             next0 = dpo0->dpoi_next_node;
 132             next1 = dpo1->dpoi_next_node;
 133             next2 = dpo2->dpoi_next_node;
 134             next3 = dpo3->dpoi_next_node;
 135
 136             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 137             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 138             vnet_buffer(b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
 139             vnet_buffer(b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
 140
 141             if (interface_count)
 142             {
 143                 vlib_increment_combined_counter (im->combined_sw_if_counters
 144                                                  + VNET_INTERFACE_COUNTER_TX,
 145                                                  thread_index,
 146                                                  adj0->rewrite_header.sw_if_index,
 147                                                  1,
 148                                                  vlib_buffer_length_in_chain (vm, b0));
 149                 vlib_increment_combined_counter (im->combined_sw_if_counters
 150                                                  + VNET_INTERFACE_COUNTER_TX,
 151                                                  thread_index,
 152                                                  adj1->rewrite_header.sw_if_index,
 153                                                  1,
 154                                                  vlib_buffer_length_in_chain (vm, b1));
 155                 vlib_increment_combined_counter (im->combined_sw_if_counters
 156                                                  + VNET_INTERFACE_COUNTER_TX,
 157                                                  thread_index,
 158                                                  adj2->rewrite_header.sw_if_index,
 159                                                  1,
 160                                                  vlib_buffer_length_in_chain (vm, b2));
 161                 vlib_increment_combined_counter (im->combined_sw_if_counters
 162                                                  + VNET_INTERFACE_COUNTER_TX,
 163                                                  thread_index,
 164                                                  adj3->rewrite_header.sw_if_index,
 165                                                  1,
 166                                                  vlib_buffer_length_in_chain (vm, b3));
 167             }
 168
 169             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 170             {
 171                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 172                                                               b0, sizeof (*tr));
 173                 tr->ai = adj_index0;
 174             }
 175             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 176             {
 177                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 178                                                               b1, sizeof (*tr));
 179                 tr->ai = adj_index1;
 180             }
 181             if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
 182             {
 183                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 184                                                               b2, sizeof (*tr));
 185                 tr->ai = adj_index2;
 186             }
 187             if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
 188             {
 189                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 190                                                               b3, sizeof (*tr));
 191                 tr->ai = adj_index3;
 192             }
 193
 194             vlib_validate_buffer_enqueue_x4 (vm, node, next_index,
 195                                              to_next, n_left_to_next,
 196                                              bi0, bi1, bi2, bi3,
 197                                              next0, next1, next2, next3);
 198         }
 199         while (n_left_from > 0 && n_left_to_next > 0)
 200         {
 201             u32 bi0, adj_index0, next0;
 202             const ip_adjacency_t * adj0;
 203             const dpo_id_t *dpo0;
 204             vlib_buffer_t * b0;
 205
 206             bi0 = from[0];
 207             to_next[0] = bi0;
 208             from += 1;
 209             to_next += 1;
 210             n_left_from -= 1;
 211             n_left_to_next -= 1;
 212
 213             b0 = vlib_get_buffer(vm, bi0);
 214
 215             /* Follow the DPO on which the midchain is stacked */
 216             adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 217             adj0 = adj_get(adj_index0);
 218             dpo0 = &adj0->sub_type.midchain.next_dpo;
 219             next0 = dpo0->dpoi_next_node;
 220             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 221
 222             if (interface_count)
 223             {
 224                 vlib_increment_combined_counter (im->combined_sw_if_counters
 225                                                  + VNET_INTERFACE_COUNTER_TX,
 226                                                  thread_index,
 227                                                  adj0->rewrite_header.sw_if_index,
 228                                                  1,
 229                                                  vlib_buffer_length_in_chain (vm, b0));
 230             }
 231
 232             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 233             {
 234                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 235                                                               b0, sizeof (*tr));
 236                 tr->ai = adj_index0;
 237             }
 238
 239             vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 240                                              to_next, n_left_to_next,
 241                                              bi0, next0);
 242         }
 243
 244         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 245     }
 246
 247     return frame->n_vectors;
 248 }
 249
 250 static u8 *
 251 format_adj_midchain_tx_trace (u8 * s, va_list * args)
 252 {
 253     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 254     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 255     adj_midchain_tx_trace_t *tr = va_arg (*args, adj_midchain_tx_trace_t*);
 256
 257     s = format(s, "adj-midchain:[%d]:%U", tr->ai,
 258                format_ip_adjacency, tr->ai,
 259                FORMAT_IP_ADJACENCY_NONE);
 260
 261     return (s);
 262 }
 263
 264 static uword
 265 adj_midchain_tx (vlib_main_t * vm,
 266                  vlib_node_runtime_t * node,
 267                  vlib_frame_t * frame)
 268 {
 269     return (adj_midchain_tx_inline(vm, node, frame, 1));
 270 }
 271
 272 VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = {
 273     .function = adj_midchain_tx,
 274     .name = "adj-midchain-tx",
 275     .vector_size = sizeof (u32),
 276
 277     .format_trace = format_adj_midchain_tx_trace,
 278
 279     .n_next_nodes = 1,
 280     .next_nodes = {
 281         [0] = "error-drop",
 282     },
 283 };
 284
 285 static uword
 286 adj_midchain_tx_no_count (vlib_main_t * vm,
 287                           vlib_node_runtime_t * node,
 288                           vlib_frame_t * frame)
 289 {
 290     return (adj_midchain_tx_inline(vm, node, frame, 0));
 291 }
 292
 293 VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = {
 294     .function = adj_midchain_tx_no_count,
 295     .name = "adj-midchain-tx-no-count",
 296     .vector_size = sizeof (u32),
 297
 298     .format_trace = format_adj_midchain_tx_trace,
 299
 300     .n_next_nodes = 1,
 301     .next_nodes = {
 302         [0] = "error-drop",
 303     },
 304 };
 305
 306 VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = {
 307     .arc_name = "ip4-output",
 308     .node_name = "adj-midchain-tx",
 309     .runs_before = VNET_FEATURES ("interface-output"),
 310     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4],
 311 };
 312 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = {
 313     .arc_name = "ip4-output",
 314     .node_name = "adj-midchain-tx-no-count",
 315     .runs_before = VNET_FEATURES ("interface-output"),
 316     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4],
 317 };
 318 VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = {
 319     .arc_name = "ip6-output",
 320     .node_name = "adj-midchain-tx",
 321     .runs_before = VNET_FEATURES ("interface-output"),
 322     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6],
 323 };
 324 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = {
 325     .arc_name = "ip6-output",
 326     .node_name = "adj-midchain-tx-no-count",
 327     .runs_before = VNET_FEATURES ("interface-output"),
 328     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6],
 329 };
 330 VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = {
 331     .arc_name = "mpls-output",
 332     .node_name = "adj-midchain-tx",
 333     .runs_before = VNET_FEATURES ("interface-output"),
 334     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS],
 335 };
 336 VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = {
 337     .arc_name = "mpls-output",
 338     .node_name = "adj-midchain-tx-no-count",
 339     .runs_before = VNET_FEATURES ("interface-output"),
 340     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS],
 341 };
 342 VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = {
 343     .arc_name = "ethernet-output",
 344     .node_name = "adj-midchain-tx",
 345     .runs_before = VNET_FEATURES ("error-drop"),
 346     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET],
 347 };
 348 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = {
 349     .arc_name = "ethernet-output",
 350     .node_name = "adj-midchain-tx-no-count",
 351     .runs_before = VNET_FEATURES ("error-drop"),
 352     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET],
 353 };
 354 VNET_FEATURE_INIT (adj_midchain_tx_nsh, static) = {
 355     .arc_name = "nsh-output",
 356     .node_name = "adj-midchain-tx",
 357     .runs_before = VNET_FEATURES ("error-drop"),
 358     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_NSH],
 359 };
 360 VNET_FEATURE_INIT (adj_midchain_tx_no_count_nsh, static) = {
 361     .arc_name = "nsh-output",
 362     .node_name = "adj-midchain-tx-no-count",
 363     .runs_before = VNET_FEATURES ("error-drop"),
 364     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_NSH],
 365 };
 366
 367 static inline u32
 368 adj_get_midchain_node (vnet_link_t link)
 369 {
 370     switch (link) {
 371     case VNET_LINK_IP4:
 372         return (ip4_midchain_node.index);
 373     case VNET_LINK_IP6:
 374         return (ip6_midchain_node.index);
 375     case VNET_LINK_MPLS:
 376         return (mpls_midchain_node.index);
 377     case VNET_LINK_ETHERNET:
 378         return (adj_l2_midchain_node.index);
 379     case VNET_LINK_NSH:
 380         return (adj_nsh_midchain_node.index);
 381     case VNET_LINK_ARP:
 382         break;
 383     }
 384     ASSERT(0);
 385     return (0);
 386 }
 387
 388 static u8
 389 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
 390 {
 391     u8 arc = (u8) ~0;
 392     switch (adj->ia_link)
 393     {
 394     case VNET_LINK_IP4:
 395         {
 396             arc = ip4_main.lookup_main.output_feature_arc_index;
 397             break;
 398         }
 399     case VNET_LINK_IP6:
 400         {
 401             arc = ip6_main.lookup_main.output_feature_arc_index;
 402             break;
 403         }
 404     case VNET_LINK_MPLS:
 405         {
 406             arc = mpls_main.output_feature_arc_index;
 407             break;
 408         }
 409     case VNET_LINK_ETHERNET:
 410         {
 411             arc = ethernet_main.output_feature_arc_index;
 412             break;
 413         }
 414     case VNET_LINK_NSH:
 415         {
 416           arc = nsh_main_dummy.output_feature_arc_index;
 417           break;
 418         }
 419     case VNET_LINK_ARP:
 420         ASSERT(0);
 421         break;
 422     }
 423
 424     ASSERT (arc != (u8) ~0);
 425
 426     return (arc);
 427 }
 428
 429 static u32
 430 adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj)
 431 {
 432     return ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
 433             adj_midchain_tx_no_count_node.index :
 434             adj_midchain_tx_node.index);
 435 }
 436
 437 static u32
 438 adj_nbr_midchain_get_feature_node (ip_adjacency_t *adj)
 439 {
 440     if (adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT)
 441     {
 442         return (adj_midchain_tx_no_count_feature_node[adj->ia_link]);
 443     }
 444
 445     return (adj_midchain_tx_feature_node[adj->ia_link]);
 446 }
 447
 448 /**
 449  * adj_midchain_setup
 450  *
 451  * Setup the adj as a mid-chain
 452  */
 453 void
 454 adj_midchain_setup (adj_index_t adj_index,
 455                     adj_midchain_fixup_t fixup,
 456                     const void *data,
 457                     adj_flags_t flags)
 458 {
 459     u32 feature_index, tx_node;
 460     ip_adjacency_t *adj;
 461     u8 arc_index;
 462
 463     ASSERT(ADJ_INDEX_INVALID != adj_index);
 464
 465     adj = adj_get(adj_index);
 466
 467     adj->sub_type.midchain.fixup_func = fixup;
 468     adj->sub_type.midchain.fixup_data = data;
 469     adj->sub_type.midchain.fei = FIB_NODE_INDEX_INVALID;
 470     adj->ia_flags |= flags;
 471
 472     arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
 473     feature_index = adj_nbr_midchain_get_feature_node(adj);
 474     tx_node = adj_nbr_midchain_get_tx_node(adj);
 475
 476     vnet_feature_enable_disable_with_index (arc_index, feature_index,
 477                                             adj->rewrite_header.sw_if_index,
 478                                             1 /* enable */, 0, 0);
 479
 480     /*
 481      * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
 482      * The graph arc used/created here is from the midchain-tx node to the
 483      * child's registered node. This is because post adj processing the next
 484      * node are any output features, then the midchain-tx.  from there we
 485      * need to get to the stacked child's node.
 486      */
 487     dpo_stack_from_node(tx_node,
 488                         &adj->sub_type.midchain.next_dpo,
 489                         drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 490 }
 491
 492 /**
 493  * adj_nbr_midchain_update_rewrite
 494  *
 495  * Update the adjacency's rewrite string. A NULL string implies the
 496  * rewrite is reset (i.e. when ARP/ND entry is gone).
 497  * NB: the adj being updated may be handling traffic in the DP.
 498  */
 499 void
 500 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
 501                                  adj_midchain_fixup_t fixup,
 502                                  const void *fixup_data,
 503                                  adj_flags_t flags,
 504                                  u8 *rewrite)
 505 {
 506     ip_adjacency_t *adj;
 507
 508     ASSERT(ADJ_INDEX_INVALID != adj_index);
 509
 510     adj = adj_get(adj_index);
 511
 512     /*
 513      * one time only update. since we don't support changing the tunnel
 514      * src,dst, this is all we need.
 515      */
 516     if (adj->lookup_next_index != IP_LOOKUP_NEXT_MIDCHAIN ||
 517         adj->lookup_next_index != IP_LOOKUP_NEXT_MCAST_MIDCHAIN)
 518     {
 519         adj_midchain_setup(adj_index, fixup, fixup_data, flags);
 520     }
 521
 522     /*
 523      * update the rewrite with the workers paused.
 524      */
 525     adj_nbr_update_rewrite_internal(adj,
 526                                     IP_LOOKUP_NEXT_MIDCHAIN,
 527                                     adj_get_midchain_node(adj->ia_link),
 528                                     adj_nbr_midchain_get_tx_node(adj),
 529                                     rewrite);
 530 }
 531
 532 /**
 533  * adj_nbr_midchain_unstack
 534  *
 535  * Unstack the adj. stack it on drop
 536  */
 537 void
 538 adj_nbr_midchain_unstack (adj_index_t adj_index)
 539 {
 540     fib_node_index_t *entry_indicies, tmp;
 541     ip_adjacency_t *adj;
 542
 543     ASSERT(ADJ_INDEX_INVALID != adj_index);
 544     adj = adj_get (adj_index);
 545
 546     /*
 547      * check to see if this unstacking breaks a recursion loop
 548      */
 549     entry_indicies = NULL;
 550     tmp = adj->sub_type.midchain.fei;
 551     adj->sub_type.midchain.fei = FIB_NODE_INDEX_INVALID;
 552
 553     if (FIB_NODE_INDEX_INVALID != tmp)
 554     {
 555         fib_entry_recursive_loop_detect(tmp, &entry_indicies);
 556         vec_free(entry_indicies);
 557     }
 558
 559     /*
 560      * stack on the drop
 561      */
 562     dpo_stack(DPO_ADJACENCY_MIDCHAIN,
 563               vnet_link_to_dpo_proto(adj->ia_link),
 564               &adj->sub_type.midchain.next_dpo,
 565               drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 566     CLIB_MEMORY_BARRIER();
 567 }
 568
 569 void
 570 adj_nbr_midchain_stack_on_fib_entry (adj_index_t ai,
 571                                      fib_node_index_t fei,
 572                                      fib_forward_chain_type_t fct)
 573 {
 574     fib_node_index_t *entry_indicies;
 575     dpo_id_t tmp = DPO_INVALID;
 576     ip_adjacency_t *adj;
 577
 578     adj = adj_get (ai);
 579
 580     /*
 581      * check to see if this stacking will form a recursion loop
 582      */
 583     entry_indicies = NULL;
 584     adj->sub_type.midchain.fei = fei;
 585
 586     if (fib_entry_recursive_loop_detect(adj->sub_type.midchain.fei, &entry_indicies))
 587     {
 588         /*
 589          * loop formed, stack on the drop.
 590          */
 591         dpo_copy(&tmp, drop_dpo_get(fib_forw_chain_type_to_dpo_proto(fct)));
 592     }
 593     else
 594     {
 595         fib_entry_contribute_forwarding (fei, fct, &tmp);
 596
 597         if ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_IP_STACK) &&
 598             (DPO_LOAD_BALANCE == tmp.dpoi_type))
 599         {
 600             /*
 601              * do that hash now and stack on the choice.
 602              * If the choice is an incomplete adj then we will need a poke when
 603              * it becomes complete. This happens since the adj update walk propagates
 604              * as far a recursive paths.
 605              */
 606             const dpo_id_t *choice;
 607             load_balance_t *lb;
 608             int hash;
 609
 610             lb = load_balance_get (tmp.dpoi_index);
 611
 612             if (FIB_FORW_CHAIN_TYPE_UNICAST_IP4 == fct)
 613             {
 614                 hash = ip4_compute_flow_hash ((ip4_header_t *) adj_get_rewrite (ai),
 615                                               lb->lb_hash_config);
 616             }
 617             else if (FIB_FORW_CHAIN_TYPE_UNICAST_IP6 == fct)
 618             {
 619                 hash = ip6_compute_flow_hash ((ip6_header_t *) adj_get_rewrite (ai),
 620                                               lb->lb_hash_config);
 621             }
 622             else
 623             {
 624                 hash = 0;
 625                 ASSERT(0);
 626             }
 627
 628             choice = load_balance_get_bucket_i (lb, hash & lb->lb_n_buckets_minus_1);
 629             dpo_copy (&tmp, choice);
 630         }
 631     }
 632     adj_nbr_midchain_stack (ai, &tmp);
 633     dpo_reset(&tmp);
 634     vec_free(entry_indicies);
 635 }
 636
 637 /**
 638  * adj_nbr_midchain_stack
 639  */
 640 void
 641 adj_nbr_midchain_stack (adj_index_t adj_index,
 642                         const dpo_id_t *next)
 643 {
 644     ip_adjacency_t *adj;
 645
 646     ASSERT(ADJ_INDEX_INVALID != adj_index);
 647
 648     adj = adj_get(adj_index);
 649
 650     ASSERT((IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index) ||
 651            (IP_LOOKUP_NEXT_MCAST_MIDCHAIN == adj->lookup_next_index));
 652
 653     dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj),
 654                         &adj->sub_type.midchain.next_dpo,
 655                         next);
 656 }
 657
 658 int
 659 adj_ndr_midchain_recursive_loop_detect (adj_index_t ai,
 660                                         fib_node_index_t **entry_indicies)
 661 {
 662     fib_node_index_t *entry_index, *entries;
 663     ip_adjacency_t * adj;
 664
 665     adj = adj_get(ai);
 666     entries = *entry_indicies;
 667
 668     vec_foreach(entry_index, entries)
 669     {
 670         if (*entry_index == adj->sub_type.midchain.fei)
 671         {
 672             /*
 673              * The entry this midchain links to is already in the set
 674              * of visited entries, this is a loop
 675              */
 676             adj->ia_flags |= ADJ_FLAG_MIDCHAIN_LOOPED;
 677             return (1);
 678         }
 679     }
 680
 681     adj->ia_flags &= ~ADJ_FLAG_MIDCHAIN_LOOPED;
 682     return (0);
 683 }
 684
 685 u8*
 686 format_adj_midchain (u8* s, va_list *ap)
 687 {
 688     index_t index = va_arg(*ap, index_t);
 689     u32 indent = va_arg(*ap, u32);
 690     ip_adjacency_t * adj = adj_get(index);
 691
 692     s = format (s, "%U", format_vnet_link, adj->ia_link);
 693     if (adj->rewrite_header.flags & VNET_REWRITE_HAS_FEATURES)
 694         s = format(s, " [features]");
 695     s = format (s, " via %U",
 696                 format_ip46_address, &adj->sub_type.nbr.next_hop,
 697                 adj_proto_to_46(adj->ia_nh_proto));
 698     s = format (s, " %U",
 699                 format_vnet_rewrite,
 700                 &adj->rewrite_header, sizeof (adj->rewrite_data), indent);
 701     s = format (s, "\n%Ustacked-on",
 702                 format_white_space, indent);
 703
 704     if (FIB_NODE_INDEX_INVALID != adj->sub_type.midchain.fei)
 705     {
 706         s = format (s, " entry:%d", adj->sub_type.midchain.fei);
 707
 708     }
 709     s = format (s, ":\n%U%U",
 710                 format_white_space, indent+2,
 711                 format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
 712
 713     return (s);
 714 }
 715
 716 static void
 717 adj_dpo_lock (dpo_id_t *dpo)
 718 {
 719     adj_lock(dpo->dpoi_index);
 720 }
 721 static void
 722 adj_dpo_unlock (dpo_id_t *dpo)
 723 {
 724     adj_unlock(dpo->dpoi_index);
 725 }
 726
 727 const static dpo_vft_t adj_midchain_dpo_vft = {
 728     .dv_lock = adj_dpo_lock,
 729     .dv_unlock = adj_dpo_unlock,
 730     .dv_format = format_adj_midchain,
 731     .dv_get_urpf = adj_dpo_get_urpf,
 732 };
 733
 734 /**
 735  * @brief The per-protocol VLIB graph nodes that are assigned to a midchain
 736  *        object.
 737  *
 738  * this means that these graph nodes are ones from which a midchain is the
 739  * parent object in the DPO-graph.
 740  */
 741 const static char* const midchain_ip4_nodes[] =
 742 {
 743     "ip4-midchain",
 744     NULL,
 745 };
 746 const static char* const midchain_ip6_nodes[] =
 747 {
 748     "ip6-midchain",
 749     NULL,
 750 };
 751 const static char* const midchain_mpls_nodes[] =
 752 {
 753     "mpls-midchain",
 754     NULL,
 755 };
 756 const static char* const midchain_ethernet_nodes[] =
 757 {
 758     "adj-l2-midchain",
 759     NULL,
 760 };
 761 const static char* const midchain_nsh_nodes[] =
 762 {
 763     "adj-nsh-midchain",
 764     NULL,
 765 };
 766
 767 const static char* const * const midchain_nodes[DPO_PROTO_NUM] =
 768 {
 769     [DPO_PROTO_IP4]  = midchain_ip4_nodes,
 770     [DPO_PROTO_IP6]  = midchain_ip6_nodes,
 771     [DPO_PROTO_MPLS] = midchain_mpls_nodes,
 772     [DPO_PROTO_ETHERNET] = midchain_ethernet_nodes,
 773     [DPO_PROTO_NSH] = midchain_nsh_nodes,
 774 };
 775
 776 void
 777 adj_midchain_module_init (void)
 778 {
 779     dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes);
 780 }