src/vnet/adj/adj_midchain.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/adj/adj_nbr.h>
  17 #include <vnet/adj/adj_internal.h>
  18 #include <vnet/adj/adj_l2.h>
  19 #include <vnet/adj/adj_nsh.h>
  20 #include <vnet/adj/adj_midchain.h>
  21 #include <vnet/ethernet/arp_packet.h>
  22 #include <vnet/dpo/drop_dpo.h>
  23 #include <vnet/dpo/load_balance.h>
  24 #include <vnet/fib/fib_walk.h>
  25 #include <vnet/fib/fib_entry.h>
  26
  27 /**
  28  * The two midchain tx feature node indices
  29  */
  30 static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM];
  31 static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM];
  32
  33 /**
  34  * @brief Trace data for packets traversing the midchain tx node
  35  */
  36 typedef struct adj_midchain_tx_trace_t_
  37 {
  38     /**
  39      * @brief the midchain adj we are traversing
  40      */
  41     adj_index_t ai;
  42 } adj_midchain_tx_trace_t;
  43
  44 always_inline uword
  45 adj_midchain_tx_inline (vlib_main_t * vm,
  46                         vlib_node_runtime_t * node,
  47                         vlib_frame_t * frame,
  48                         int interface_count)
  49 {
  50     u32 * from, * to_next, n_left_from, n_left_to_next;
  51     u32 next_index;
  52     vnet_main_t *vnm = vnet_get_main ();
  53     vnet_interface_main_t *im = &vnm->interface_main;
  54     u32 thread_index = vm->thread_index;
  55
  56     /* Vector of buffer / pkt indices we're supposed to process */
  57     from = vlib_frame_vector_args (frame);
  58
  59     /* Number of buffers / pkts */
  60     n_left_from = frame->n_vectors;
  61
  62     /* Speculatively send the first buffer to the last disposition we used */
  63     next_index = node->cached_next_index;
  64
  65     while (n_left_from > 0)
  66     {
  67         /* set up to enqueue to our disposition with index = next_index */
  68         vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
  69
  70         while (n_left_from >= 8 && n_left_to_next > 4)
  71         {
  72             u32 bi0, adj_index0, next0;
  73             const ip_adjacency_t * adj0;
  74             const dpo_id_t *dpo0;
  75             vlib_buffer_t * b0;
  76             u32 bi1, adj_index1, next1;
  77             const ip_adjacency_t * adj1;
  78             const dpo_id_t *dpo1;
  79             vlib_buffer_t * b1;
  80             u32 bi2, adj_index2, next2;
  81             const ip_adjacency_t * adj2;
  82             const dpo_id_t *dpo2;
  83             vlib_buffer_t * b2;
  84             u32 bi3, adj_index3, next3;
  85             const ip_adjacency_t * adj3;
  86             const dpo_id_t *dpo3;
  87             vlib_buffer_t * b3;
  88
  89             /* Prefetch next iteration. */
  90             {
  91                 vlib_buffer_t * p4, * p5;
  92                 vlib_buffer_t * p6, * p7;
  93
  94                 p4 = vlib_get_buffer (vm, from[4]);
  95                 p5 = vlib_get_buffer (vm, from[5]);
  96                 p6 = vlib_get_buffer (vm, from[6]);
  97                 p7 = vlib_get_buffer (vm, from[7]);
  98
  99                 vlib_prefetch_buffer_header (p4, LOAD);
 100                 vlib_prefetch_buffer_header (p5, LOAD);
 101                 vlib_prefetch_buffer_header (p6, LOAD);
 102                 vlib_prefetch_buffer_header (p7, LOAD);
 103             }
 104
 105             bi0 = from[0];
 106             to_next[0] = bi0;
 107             bi1 = from[1];
 108             to_next[1] = bi1;
 109             bi2 = from[2];
 110             to_next[2] = bi2;
 111             bi3 = from[3];
 112             to_next[3] = bi3;
 113
 114             from += 4;
 115             to_next += 4;
 116             n_left_from -= 4;
 117             n_left_to_next -= 4;
 118
 119             b0 = vlib_get_buffer(vm, bi0);
 120             b1 = vlib_get_buffer(vm, bi1);
 121             b2 = vlib_get_buffer(vm, bi2);
 122             b3 = vlib_get_buffer(vm, bi3);
 123
 124             /* Follow the DPO on which the midchain is stacked */
 125             adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 126             adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 127             adj_index2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
 128             adj_index3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
 129
 130             adj0 = adj_get(adj_index0);
 131             adj1 = adj_get(adj_index1);
 132             adj2 = adj_get(adj_index2);
 133             adj3 = adj_get(adj_index3);
 134
 135             dpo0 = &adj0->sub_type.midchain.next_dpo;
 136             dpo1 = &adj1->sub_type.midchain.next_dpo;
 137             dpo2 = &adj2->sub_type.midchain.next_dpo;
 138             dpo3 = &adj3->sub_type.midchain.next_dpo;
 139
 140             next0 = dpo0->dpoi_next_node;
 141             next1 = dpo1->dpoi_next_node;
 142             next2 = dpo2->dpoi_next_node;
 143             next3 = dpo3->dpoi_next_node;
 144
 145             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 146             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 147             vnet_buffer(b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
 148             vnet_buffer(b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
 149
 150             if (interface_count)
 151             {
 152                 vlib_increment_combined_counter (im->combined_sw_if_counters
 153                                                  + VNET_INTERFACE_COUNTER_TX,
 154                                                  thread_index,
 155                                                  adj0->rewrite_header.sw_if_index,
 156                                                  1,
 157                                                  vlib_buffer_length_in_chain (vm, b0));
 158                 vlib_increment_combined_counter (im->combined_sw_if_counters
 159                                                  + VNET_INTERFACE_COUNTER_TX,
 160                                                  thread_index,
 161                                                  adj1->rewrite_header.sw_if_index,
 162                                                  1,
 163                                                  vlib_buffer_length_in_chain (vm, b1));
 164                 vlib_increment_combined_counter (im->combined_sw_if_counters
 165                                                  + VNET_INTERFACE_COUNTER_TX,
 166                                                  thread_index,
 167                                                  adj2->rewrite_header.sw_if_index,
 168                                                  1,
 169                                                  vlib_buffer_length_in_chain (vm, b2));
 170                 vlib_increment_combined_counter (im->combined_sw_if_counters
 171                                                  + VNET_INTERFACE_COUNTER_TX,
 172                                                  thread_index,
 173                                                  adj3->rewrite_header.sw_if_index,
 174                                                  1,
 175                                                  vlib_buffer_length_in_chain (vm, b3));
 176             }
 177
 178             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 179             {
 180                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 181                                                               b0, sizeof (*tr));
 182                 tr->ai = adj_index0;
 183             }
 184             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 185             {
 186                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 187                                                               b1, sizeof (*tr));
 188                 tr->ai = adj_index1;
 189             }
 190             if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
 191             {
 192                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 193                                                               b2, sizeof (*tr));
 194                 tr->ai = adj_index2;
 195             }
 196             if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
 197             {
 198                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 199                                                               b3, sizeof (*tr));
 200                 tr->ai = adj_index3;
 201             }
 202
 203             vlib_validate_buffer_enqueue_x4 (vm, node, next_index,
 204                                              to_next, n_left_to_next,
 205                                              bi0, bi1, bi2, bi3,
 206                                              next0, next1, next2, next3);
 207         }
 208         while (n_left_from > 0 && n_left_to_next > 0)
 209         {
 210             u32 bi0, adj_index0, next0;
 211             const ip_adjacency_t * adj0;
 212             const dpo_id_t *dpo0;
 213             vlib_buffer_t * b0;
 214
 215             bi0 = from[0];
 216             to_next[0] = bi0;
 217             from += 1;
 218             to_next += 1;
 219             n_left_from -= 1;
 220             n_left_to_next -= 1;
 221
 222             b0 = vlib_get_buffer(vm, bi0);
 223
 224             /* Follow the DPO on which the midchain is stacked */
 225             adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 226             adj0 = adj_get(adj_index0);
 227             dpo0 = &adj0->sub_type.midchain.next_dpo;
 228             next0 = dpo0->dpoi_next_node;
 229             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 230
 231             if (interface_count)
 232             {
 233                 vlib_increment_combined_counter (im->combined_sw_if_counters
 234                                                  + VNET_INTERFACE_COUNTER_TX,
 235                                                  thread_index,
 236                                                  adj0->rewrite_header.sw_if_index,
 237                                                  1,
 238                                                  vlib_buffer_length_in_chain (vm, b0));
 239             }
 240
 241             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 242             {
 243                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 244                                                               b0, sizeof (*tr));
 245                 tr->ai = adj_index0;
 246             }
 247
 248             vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 249                                              to_next, n_left_to_next,
 250                                              bi0, next0);
 251         }
 252
 253         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 254     }
 255
 256     return frame->n_vectors;
 257 }
 258
 259 static u8 *
 260 format_adj_midchain_tx_trace (u8 * s, va_list * args)
 261 {
 262     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 263     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 264     adj_midchain_tx_trace_t *tr = va_arg (*args, adj_midchain_tx_trace_t*);
 265
 266     s = format(s, "adj-midchain:[%d]:%U", tr->ai,
 267                format_ip_adjacency, tr->ai,
 268                FORMAT_IP_ADJACENCY_NONE);
 269
 270     return (s);
 271 }
 272
 273 static uword
 274 adj_midchain_tx (vlib_main_t * vm,
 275                  vlib_node_runtime_t * node,
 276                  vlib_frame_t * frame)
 277 {
 278     return (adj_midchain_tx_inline(vm, node, frame, 1));
 279 }
 280
 281 VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = {
 282     .function = adj_midchain_tx,
 283     .name = "adj-midchain-tx",
 284     .vector_size = sizeof (u32),
 285
 286     .format_trace = format_adj_midchain_tx_trace,
 287
 288     .n_next_nodes = 1,
 289     .next_nodes = {
 290         [0] = "error-drop",
 291     },
 292 };
 293
 294 static uword
 295 adj_midchain_tx_no_count (vlib_main_t * vm,
 296                           vlib_node_runtime_t * node,
 297                           vlib_frame_t * frame)
 298 {
 299     return (adj_midchain_tx_inline(vm, node, frame, 0));
 300 }
 301
 302 VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = {
 303     .function = adj_midchain_tx_no_count,
 304     .name = "adj-midchain-tx-no-count",
 305     .vector_size = sizeof (u32),
 306
 307     .format_trace = format_adj_midchain_tx_trace,
 308
 309     .n_next_nodes = 1,
 310     .next_nodes = {
 311         [0] = "error-drop",
 312     },
 313 };
 314
 315 VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = {
 316     .arc_name = "ip4-output",
 317     .node_name = "adj-midchain-tx",
 318     .runs_before = VNET_FEATURES ("interface-output"),
 319     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4],
 320 };
 321 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = {
 322     .arc_name = "ip4-output",
 323     .node_name = "adj-midchain-tx-no-count",
 324     .runs_before = VNET_FEATURES ("interface-output"),
 325     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4],
 326 };
 327 VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = {
 328     .arc_name = "ip6-output",
 329     .node_name = "adj-midchain-tx",
 330     .runs_before = VNET_FEATURES ("interface-output"),
 331     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6],
 332 };
 333 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = {
 334     .arc_name = "ip6-output",
 335     .node_name = "adj-midchain-tx-no-count",
 336     .runs_before = VNET_FEATURES ("interface-output"),
 337     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6],
 338 };
 339 VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = {
 340     .arc_name = "mpls-output",
 341     .node_name = "adj-midchain-tx",
 342     .runs_before = VNET_FEATURES ("interface-output"),
 343     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS],
 344 };
 345 VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = {
 346     .arc_name = "mpls-output",
 347     .node_name = "adj-midchain-tx-no-count",
 348     .runs_before = VNET_FEATURES ("interface-output"),
 349     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS],
 350 };
 351 VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = {
 352     .arc_name = "ethernet-output",
 353     .node_name = "adj-midchain-tx",
 354     .runs_before = VNET_FEATURES ("error-drop"),
 355     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET],
 356 };
 357 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = {
 358     .arc_name = "ethernet-output",
 359     .node_name = "adj-midchain-tx-no-count",
 360     .runs_before = VNET_FEATURES ("error-drop"),
 361     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET],
 362 };
 363 VNET_FEATURE_INIT (adj_midchain_tx_nsh, static) = {
 364     .arc_name = "nsh-output",
 365     .node_name = "adj-midchain-tx",
 366     .runs_before = VNET_FEATURES ("error-drop"),
 367     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_NSH],
 368 };
 369 VNET_FEATURE_INIT (adj_midchain_tx_no_count_nsh, static) = {
 370     .arc_name = "nsh-output",
 371     .node_name = "adj-midchain-tx-no-count",
 372     .runs_before = VNET_FEATURES ("error-drop"),
 373     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_NSH],
 374 };
 375
 376 static inline u32
 377 adj_get_midchain_node (vnet_link_t link)
 378 {
 379     switch (link) {
 380     case VNET_LINK_IP4:
 381         return (ip4_midchain_node.index);
 382     case VNET_LINK_IP6:
 383         return (ip6_midchain_node.index);
 384     case VNET_LINK_MPLS:
 385         return (mpls_midchain_node.index);
 386     case VNET_LINK_ETHERNET:
 387         return (adj_l2_midchain_node.index);
 388     case VNET_LINK_NSH:
 389         return (adj_nsh_midchain_node.index);
 390     case VNET_LINK_ARP:
 391         break;
 392     }
 393     ASSERT(0);
 394     return (0);
 395 }
 396
 397 static u8
 398 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
 399 {
 400     u8 arc = (u8) ~0;
 401     switch (adj->ia_link)
 402     {
 403     case VNET_LINK_IP4:
 404         {
 405             arc = ip4_main.lookup_main.output_feature_arc_index;
 406             break;
 407         }
 408     case VNET_LINK_IP6:
 409         {
 410             arc = ip6_main.lookup_main.output_feature_arc_index;
 411             break;
 412         }
 413     case VNET_LINK_MPLS:
 414         {
 415             arc = mpls_main.output_feature_arc_index;
 416             break;
 417         }
 418     case VNET_LINK_ETHERNET:
 419         {
 420             arc = ethernet_main.output_feature_arc_index;
 421             break;
 422         }
 423     case VNET_LINK_NSH:
 424         {
 425           arc = nsh_main_dummy.output_feature_arc_index;
 426           break;
 427         }
 428     case VNET_LINK_ARP:
 429         ASSERT(0);
 430         break;
 431     }
 432
 433     ASSERT (arc != (u8) ~0);
 434
 435     return (arc);
 436 }
 437
 438 static u32
 439 adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj)
 440 {
 441     return ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
 442             adj_midchain_tx_no_count_node.index :
 443             adj_midchain_tx_node.index);
 444 }
 445
 446 static u32
 447 adj_nbr_midchain_get_feature_node (ip_adjacency_t *adj)
 448 {
 449     if (adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT)
 450     {
 451         return (adj_midchain_tx_no_count_feature_node[adj->ia_link]);
 452     }
 453
 454     return (adj_midchain_tx_feature_node[adj->ia_link]);
 455 }
 456
 457 /**
 458  * adj_midchain_setup
 459  *
 460  * Setup the adj as a mid-chain
 461  */
 462 void
 463 adj_midchain_setup (adj_index_t adj_index,
 464                     adj_midchain_fixup_t fixup,
 465                     const void *data,
 466                     adj_flags_t flags)
 467 {
 468     u32 feature_index, tx_node;
 469     ip_adjacency_t *adj;
 470     u8 arc_index;
 471
 472     ASSERT(ADJ_INDEX_INVALID != adj_index);
 473
 474     adj = adj_get(adj_index);
 475
 476     adj->sub_type.midchain.fixup_func = fixup;
 477     adj->sub_type.midchain.fixup_data = data;
 478     adj->sub_type.midchain.fei = FIB_NODE_INDEX_INVALID;
 479     adj->ia_flags |= flags;
 480
 481     arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
 482     feature_index = adj_nbr_midchain_get_feature_node(adj);
 483     tx_node = adj_nbr_midchain_get_tx_node(adj);
 484
 485     vnet_feature_enable_disable_with_index (arc_index, feature_index,
 486                                             adj->rewrite_header.sw_if_index,
 487                                             1 /* enable */, 0, 0);
 488
 489     /*
 490      * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
 491      * The graph arc used/created here is from the midchain-tx node to the
 492      * child's registered node. This is because post adj processing the next
 493      * node are any output features, then the midchain-tx.  from there we
 494      * need to get to the stacked child's node.
 495      */
 496     dpo_stack_from_node(tx_node,
 497                         &adj->sub_type.midchain.next_dpo,
 498                         drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 499 }
 500
 501 /**
 502  * adj_nbr_midchain_update_rewrite
 503  *
 504  * Update the adjacency's rewrite string. A NULL string implies the
 505  * rewrite is reset (i.e. when ARP/ND entry is gone).
 506  * NB: the adj being updated may be handling traffic in the DP.
 507  */
 508 void
 509 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
 510                                  adj_midchain_fixup_t fixup,
 511                                  const void *fixup_data,
 512                                  adj_flags_t flags,
 513                                  u8 *rewrite)
 514 {
 515     ip_adjacency_t *adj;
 516
 517     ASSERT(ADJ_INDEX_INVALID != adj_index);
 518
 519     adj = adj_get(adj_index);
 520
 521     /*
 522      * one time only update. since we don't support changing the tunnel
 523      * src,dst, this is all we need.
 524      */
 525     ASSERT((adj->lookup_next_index == IP_LOOKUP_NEXT_ARP) ||
 526            (adj->lookup_next_index == IP_LOOKUP_NEXT_GLEAN) ||
 527            (adj->lookup_next_index == IP_LOOKUP_NEXT_BCAST));
 528
 529     /*
 530      * tunnels can always provide a rewrite.
 531      */
 532     ASSERT(NULL != rewrite);
 533
 534     adj_midchain_setup(adj_index, fixup, fixup_data, flags);
 535
 536     /*
 537      * update the rewrite with the workers paused.
 538      */
 539     adj_nbr_update_rewrite_internal(adj,
 540                                     IP_LOOKUP_NEXT_MIDCHAIN,
 541                                     adj_get_midchain_node(adj->ia_link),
 542                                     adj_nbr_midchain_get_tx_node(adj),
 543                                     rewrite);
 544 }
 545
 546 /**
 547  * adj_nbr_midchain_unstack
 548  *
 549  * Unstack the adj. stack it on drop
 550  */
 551 void
 552 adj_nbr_midchain_unstack (adj_index_t adj_index)
 553 {
 554     fib_node_index_t *entry_indicies, tmp;
 555     ip_adjacency_t *adj;
 556
 557     ASSERT(ADJ_INDEX_INVALID != adj_index);
 558     adj = adj_get (adj_index);
 559
 560     /*
 561      * check to see if this unstacking breaks a recursion loop
 562      */
 563     entry_indicies = NULL;
 564     tmp = adj->sub_type.midchain.fei;
 565     adj->sub_type.midchain.fei = FIB_NODE_INDEX_INVALID;
 566
 567     if (FIB_NODE_INDEX_INVALID != tmp)
 568     {
 569         fib_entry_recursive_loop_detect(tmp, &entry_indicies);
 570         vec_free(entry_indicies);
 571     }
 572
 573     /*
 574      * stack on the drop
 575      */
 576     dpo_stack(DPO_ADJACENCY_MIDCHAIN,
 577               vnet_link_to_dpo_proto(adj->ia_link),
 578               &adj->sub_type.midchain.next_dpo,
 579               drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 580     CLIB_MEMORY_BARRIER();
 581 }
 582
 583 void
 584 adj_nbr_midchain_stack_on_fib_entry (adj_index_t ai,
 585                                      fib_node_index_t fei,
 586                                      fib_forward_chain_type_t fct)
 587 {
 588     fib_node_index_t *entry_indicies;
 589     dpo_id_t tmp = DPO_INVALID;
 590     ip_adjacency_t *adj;
 591
 592     adj = adj_get (ai);
 593
 594     /*
 595      * check to see if this stacking will form a recursion loop
 596      */
 597     entry_indicies = NULL;
 598     adj->sub_type.midchain.fei = fei;
 599
 600     if (fib_entry_recursive_loop_detect(adj->sub_type.midchain.fei, &entry_indicies))
 601     {
 602         /*
 603          * loop formed, stack on the drop.
 604          */
 605         dpo_copy(&tmp, drop_dpo_get(fib_forw_chain_type_to_dpo_proto(fct)));
 606     }
 607     else
 608     {
 609         fib_entry_contribute_forwarding (fei, fct, &tmp);
 610
 611         if ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_IP_STACK) &&
 612             (DPO_LOAD_BALANCE == tmp.dpoi_type))
 613         {
 614             /*
 615              * do that hash now and stack on the choice.
 616              * If the choice is an incomplete adj then we will need a poke when
 617              * it becomes complete. This happens since the adj update walk propagates
 618              * as far a recursive paths.
 619              */
 620             const dpo_id_t *choice;
 621             load_balance_t *lb;
 622             int hash;
 623
 624             lb = load_balance_get (tmp.dpoi_index);
 625
 626             if (FIB_FORW_CHAIN_TYPE_UNICAST_IP4 == fct)
 627             {
 628                 hash = ip4_compute_flow_hash ((ip4_header_t *) adj_get_rewrite (ai),
 629                                               lb->lb_hash_config);
 630             }
 631             else if (FIB_FORW_CHAIN_TYPE_UNICAST_IP6 == fct)
 632             {
 633                 hash = ip6_compute_flow_hash ((ip6_header_t *) adj_get_rewrite (ai),
 634                                               lb->lb_hash_config);
 635             }
 636             else
 637             {
 638                 hash = 0;
 639                 ASSERT(0);
 640             }
 641
 642             choice = load_balance_get_bucket_i (lb, hash & lb->lb_n_buckets_minus_1);
 643             dpo_copy (&tmp, choice);
 644         }
 645     }
 646     adj_nbr_midchain_stack (ai, &tmp);
 647     dpo_reset(&tmp);
 648     vec_free(entry_indicies);
 649 }
 650
 651 /**
 652  * adj_nbr_midchain_stack
 653  */
 654 void
 655 adj_nbr_midchain_stack (adj_index_t adj_index,
 656                         const dpo_id_t *next)
 657 {
 658     ip_adjacency_t *adj;
 659
 660     ASSERT(ADJ_INDEX_INVALID != adj_index);
 661
 662     adj = adj_get(adj_index);
 663
 664     ASSERT((IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index) ||
 665            (IP_LOOKUP_NEXT_MCAST_MIDCHAIN == adj->lookup_next_index));
 666
 667     dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj),
 668                         &adj->sub_type.midchain.next_dpo,
 669                         next);
 670 }
 671
 672 int
 673 adj_ndr_midchain_recursive_loop_detect (adj_index_t ai,
 674                                         fib_node_index_t **entry_indicies)
 675 {
 676     fib_node_index_t *entry_index, *entries;
 677     ip_adjacency_t * adj;
 678
 679     adj = adj_get(ai);
 680     entries = *entry_indicies;
 681
 682     vec_foreach(entry_index, entries)
 683     {
 684         if (*entry_index == adj->sub_type.midchain.fei)
 685         {
 686             /*
 687              * The entry this midchain links to is already in the set
 688              * of visited entries, this is a loop
 689              */
 690             adj->ia_flags |= ADJ_FLAG_MIDCHAIN_LOOPED;
 691             return (1);
 692         }
 693     }
 694
 695     adj->ia_flags &= ~ADJ_FLAG_MIDCHAIN_LOOPED;
 696     return (0);
 697 }
 698
 699 u8*
 700 format_adj_midchain (u8* s, va_list *ap)
 701 {
 702     index_t index = va_arg(*ap, index_t);
 703     u32 indent = va_arg(*ap, u32);
 704     ip_adjacency_t * adj = adj_get(index);
 705
 706     s = format (s, "%U", format_vnet_link, adj->ia_link);
 707     s = format (s, " via %U",
 708                 format_ip46_address, &adj->sub_type.nbr.next_hop,
 709                 adj_proto_to_46(adj->ia_nh_proto));
 710     s = format (s, " %U",
 711                 format_vnet_rewrite,
 712                 &adj->rewrite_header, sizeof (adj->rewrite_data), indent);
 713     s = format (s, "\n%Ustacked-on",
 714                 format_white_space, indent);
 715
 716     if (FIB_NODE_INDEX_INVALID != adj->sub_type.midchain.fei)
 717     {
 718         s = format (s, " entry:%d", adj->sub_type.midchain.fei);
 719
 720     }
 721     s = format (s, ":\n%U%U",
 722                 format_white_space, indent+2,
 723                 format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
 724
 725     return (s);
 726 }
 727
 728 static void
 729 adj_dpo_lock (dpo_id_t *dpo)
 730 {
 731     adj_lock(dpo->dpoi_index);
 732 }
 733 static void
 734 adj_dpo_unlock (dpo_id_t *dpo)
 735 {
 736     adj_unlock(dpo->dpoi_index);
 737 }
 738
 739 const static dpo_vft_t adj_midchain_dpo_vft = {
 740     .dv_lock = adj_dpo_lock,
 741     .dv_unlock = adj_dpo_unlock,
 742     .dv_format = format_adj_midchain,
 743     .dv_get_urpf = adj_dpo_get_urpf,
 744 };
 745
 746 /**
 747  * @brief The per-protocol VLIB graph nodes that are assigned to a midchain
 748  *        object.
 749  *
 750  * this means that these graph nodes are ones from which a midchain is the
 751  * parent object in the DPO-graph.
 752  */
 753 const static char* const midchain_ip4_nodes[] =
 754 {
 755     "ip4-midchain",
 756     NULL,
 757 };
 758 const static char* const midchain_ip6_nodes[] =
 759 {
 760     "ip6-midchain",
 761     NULL,
 762 };
 763 const static char* const midchain_mpls_nodes[] =
 764 {
 765     "mpls-midchain",
 766     NULL,
 767 };
 768 const static char* const midchain_ethernet_nodes[] =
 769 {
 770     "adj-l2-midchain",
 771     NULL,
 772 };
 773 const static char* const midchain_nsh_nodes[] =
 774 {
 775     "adj-nsh-midchain",
 776     NULL,
 777 };
 778
 779 const static char* const * const midchain_nodes[DPO_PROTO_NUM] =
 780 {
 781     [DPO_PROTO_IP4]  = midchain_ip4_nodes,
 782     [DPO_PROTO_IP6]  = midchain_ip6_nodes,
 783     [DPO_PROTO_MPLS] = midchain_mpls_nodes,
 784     [DPO_PROTO_ETHERNET] = midchain_ethernet_nodes,
 785     [DPO_PROTO_NSH] = midchain_nsh_nodes,
 786 };
 787
 788 void
 789 adj_midchain_module_init (void)
 790 {
 791     dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes);
 792 }