src/vnet/adj/adj_midchain.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/adj/adj_nbr.h>
  17 #include <vnet/adj/adj_internal.h>
  18 #include <vnet/adj/adj_l2.h>
  19 #include <vnet/adj/adj_nsh.h>
  20 #include <vnet/adj/adj_midchain.h>
  21 #include <vnet/ethernet/arp_packet.h>
  22 #include <vnet/dpo/drop_dpo.h>
  23 #include <vnet/dpo/load_balance.h>
  24 #include <vnet/fib/fib_walk.h>
  25 #include <vnet/fib/fib_entry.h>
  26
  27 /**
  28  * The two midchain tx feature node indices
  29  */
  30 static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM];
  31 static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM];
  32
  33 static u32 *adj_midchain_feat_count_per_sw_if_index[VNET_LINK_NUM];
  34
  35 /**
  36  * @brief Trace data for packets traversing the midchain tx node
  37  */
  38 typedef struct adj_midchain_tx_trace_t_
  39 {
  40     /**
  41      * @brief the midchain adj we are traversing
  42      */
  43     adj_index_t ai;
  44 } adj_midchain_tx_trace_t;
  45
  46 always_inline uword
  47 adj_midchain_tx_inline (vlib_main_t * vm,
  48                         vlib_node_runtime_t * node,
  49                         vlib_frame_t * frame,
  50                         int interface_count)
  51 {
  52     u32 * from, * to_next, n_left_from, n_left_to_next;
  53     u32 next_index;
  54     vnet_main_t *vnm = vnet_get_main ();
  55     vnet_interface_main_t *im = &vnm->interface_main;
  56     u32 thread_index = vm->thread_index;
  57
  58     /* Vector of buffer / pkt indices we're supposed to process */
  59     from = vlib_frame_vector_args (frame);
  60
  61     /* Number of buffers / pkts */
  62     n_left_from = frame->n_vectors;
  63
  64     /* Speculatively send the first buffer to the last disposition we used */
  65     next_index = node->cached_next_index;
  66
  67     while (n_left_from > 0)
  68     {
  69         /* set up to enqueue to our disposition with index = next_index */
  70         vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
  71
  72         while (n_left_from >= 8 && n_left_to_next > 4)
  73         {
  74             const ip_adjacency_t *adj0, *adj1, *adj2, *adj3;
  75             const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3;
  76             vlib_buffer_t * b0, *b1, *b2, *b3;
  77             u32 bi0, adj_index0, next0;
  78             u32 bi1, adj_index1, next1;
  79             u32 bi2, adj_index2, next2;
  80             u32 bi3, adj_index3, next3;
  81
  82             /* Prefetch next iteration. */
  83             {
  84                 vlib_buffer_t * p4, * p5;
  85                 vlib_buffer_t * p6, * p7;
  86
  87                 p4 = vlib_get_buffer (vm, from[4]);
  88                 p5 = vlib_get_buffer (vm, from[5]);
  89                 p6 = vlib_get_buffer (vm, from[6]);
  90                 p7 = vlib_get_buffer (vm, from[7]);
  91
  92                 vlib_prefetch_buffer_header (p4, LOAD);
  93                 vlib_prefetch_buffer_header (p5, LOAD);
  94                 vlib_prefetch_buffer_header (p6, LOAD);
  95                 vlib_prefetch_buffer_header (p7, LOAD);
  96             }
  97
  98             bi0 = from[0];
  99             to_next[0] = bi0;
 100             bi1 = from[1];
 101             to_next[1] = bi1;
 102             bi2 = from[2];
 103             to_next[2] = bi2;
 104             bi3 = from[3];
 105             to_next[3] = bi3;
 106
 107             from += 4;
 108             to_next += 4;
 109             n_left_from -= 4;
 110             n_left_to_next -= 4;
 111
 112             b0 = vlib_get_buffer(vm, bi0);
 113             b1 = vlib_get_buffer(vm, bi1);
 114             b2 = vlib_get_buffer(vm, bi2);
 115             b3 = vlib_get_buffer(vm, bi3);
 116
 117             /* Follow the DPO on which the midchain is stacked */
 118             adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 119             adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 120             adj_index2 = vnet_buffer(b2)->ip.adj_index[VLIB_TX];
 121             adj_index3 = vnet_buffer(b3)->ip.adj_index[VLIB_TX];
 122
 123             adj0 = adj_get(adj_index0);
 124             adj1 = adj_get(adj_index1);
 125             adj2 = adj_get(adj_index2);
 126             adj3 = adj_get(adj_index3);
 127
 128             dpo0 = &adj0->sub_type.midchain.next_dpo;
 129             dpo1 = &adj1->sub_type.midchain.next_dpo;
 130             dpo2 = &adj2->sub_type.midchain.next_dpo;
 131             dpo3 = &adj3->sub_type.midchain.next_dpo;
 132
 133             next0 = dpo0->dpoi_next_node;
 134             next1 = dpo1->dpoi_next_node;
 135             next2 = dpo2->dpoi_next_node;
 136             next3 = dpo3->dpoi_next_node;
 137
 138             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 139             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 140             vnet_buffer(b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
 141             vnet_buffer(b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
 142
 143             if (interface_count)
 144             {
 145                 vlib_increment_combined_counter (im->combined_sw_if_counters
 146                                                  + VNET_INTERFACE_COUNTER_TX,
 147                                                  thread_index,
 148                                                  adj0->rewrite_header.sw_if_index,
 149                                                  1,
 150                                                  vlib_buffer_length_in_chain (vm, b0));
 151                 vlib_increment_combined_counter (im->combined_sw_if_counters
 152                                                  + VNET_INTERFACE_COUNTER_TX,
 153                                                  thread_index,
 154                                                  adj1->rewrite_header.sw_if_index,
 155                                                  1,
 156                                                  vlib_buffer_length_in_chain (vm, b1));
 157                 vlib_increment_combined_counter (im->combined_sw_if_counters
 158                                                  + VNET_INTERFACE_COUNTER_TX,
 159                                                  thread_index,
 160                                                  adj2->rewrite_header.sw_if_index,
 161                                                  1,
 162                                                  vlib_buffer_length_in_chain (vm, b2));
 163                 vlib_increment_combined_counter (im->combined_sw_if_counters
 164                                                  + VNET_INTERFACE_COUNTER_TX,
 165                                                  thread_index,
 166                                                  adj3->rewrite_header.sw_if_index,
 167                                                  1,
 168                                                  vlib_buffer_length_in_chain (vm, b3));
 169             }
 170
 171             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 172             {
 173                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 174                                                               b0, sizeof (*tr));
 175                 tr->ai = adj_index0;
 176             }
 177             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 178             {
 179                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 180                                                               b1, sizeof (*tr));
 181                 tr->ai = adj_index1;
 182             }
 183             if (PREDICT_FALSE(b2->flags & VLIB_BUFFER_IS_TRACED))
 184             {
 185                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 186                                                               b2, sizeof (*tr));
 187                 tr->ai = adj_index2;
 188             }
 189             if (PREDICT_FALSE(b3->flags & VLIB_BUFFER_IS_TRACED))
 190             {
 191                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 192                                                               b3, sizeof (*tr));
 193                 tr->ai = adj_index3;
 194             }
 195
 196             vlib_validate_buffer_enqueue_x4 (vm, node, next_index,
 197                                              to_next, n_left_to_next,
 198                                              bi0, bi1, bi2, bi3,
 199                                              next0, next1, next2, next3);
 200         }
 201         while (n_left_from > 0 && n_left_to_next > 0)
 202         {
 203             u32 bi0, adj_index0, next0;
 204             const ip_adjacency_t * adj0;
 205             const dpo_id_t *dpo0;
 206             vlib_buffer_t * b0;
 207
 208             bi0 = from[0];
 209             to_next[0] = bi0;
 210             from += 1;
 211             to_next += 1;
 212             n_left_from -= 1;
 213             n_left_to_next -= 1;
 214
 215             b0 = vlib_get_buffer(vm, bi0);
 216
 217             /* Follow the DPO on which the midchain is stacked */
 218             adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 219             adj0 = adj_get(adj_index0);
 220             dpo0 = &adj0->sub_type.midchain.next_dpo;
 221             next0 = dpo0->dpoi_next_node;
 222             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 223
 224             if (interface_count)
 225             {
 226                 vlib_increment_combined_counter (im->combined_sw_if_counters
 227                                                  + VNET_INTERFACE_COUNTER_TX,
 228                                                  thread_index,
 229                                                  adj0->rewrite_header.sw_if_index,
 230                                                  1,
 231                                                  vlib_buffer_length_in_chain (vm, b0));
 232             }
 233
 234             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 235             {
 236                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 237                                                               b0, sizeof (*tr));
 238                 tr->ai = adj_index0;
 239             }
 240
 241             vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 242                                              to_next, n_left_to_next,
 243                                              bi0, next0);
 244         }
 245
 246         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 247     }
 248
 249     return frame->n_vectors;
 250 }
 251
 252 static u8 *
 253 format_adj_midchain_tx_trace (u8 * s, va_list * args)
 254 {
 255     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 256     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 257     adj_midchain_tx_trace_t *tr = va_arg (*args, adj_midchain_tx_trace_t*);
 258
 259     s = format(s, "adj-midchain:[%d]:%U", tr->ai,
 260                format_ip_adjacency, tr->ai,
 261                FORMAT_IP_ADJACENCY_NONE);
 262
 263     return (s);
 264 }
 265
 266 static uword
 267 adj_midchain_tx (vlib_main_t * vm,
 268                  vlib_node_runtime_t * node,
 269                  vlib_frame_t * frame)
 270 {
 271     return (adj_midchain_tx_inline(vm, node, frame, 1));
 272 }
 273
 274 VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = {
 275     .function = adj_midchain_tx,
 276     .name = "adj-midchain-tx",
 277     .vector_size = sizeof (u32),
 278
 279     .format_trace = format_adj_midchain_tx_trace,
 280
 281     .n_next_nodes = 1,
 282     .next_nodes = {
 283         [0] = "error-drop",
 284     },
 285 };
 286
 287 static uword
 288 adj_midchain_tx_no_count (vlib_main_t * vm,
 289                           vlib_node_runtime_t * node,
 290                           vlib_frame_t * frame)
 291 {
 292     return (adj_midchain_tx_inline(vm, node, frame, 0));
 293 }
 294
 295 VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = {
 296     .function = adj_midchain_tx_no_count,
 297     .name = "adj-midchain-tx-no-count",
 298     .vector_size = sizeof (u32),
 299
 300     .format_trace = format_adj_midchain_tx_trace,
 301
 302     .n_next_nodes = 1,
 303     .next_nodes = {
 304         [0] = "error-drop",
 305     },
 306 };
 307
 308 VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = {
 309     .arc_name = "ip4-output",
 310     .node_name = "adj-midchain-tx",
 311     .runs_before = VNET_FEATURES ("interface-output"),
 312     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4],
 313 };
 314 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = {
 315     .arc_name = "ip4-output",
 316     .node_name = "adj-midchain-tx-no-count",
 317     .runs_before = VNET_FEATURES ("interface-output"),
 318     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4],
 319 };
 320 VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = {
 321     .arc_name = "ip6-output",
 322     .node_name = "adj-midchain-tx",
 323     .runs_before = VNET_FEATURES ("interface-output"),
 324     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6],
 325 };
 326 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = {
 327     .arc_name = "ip6-output",
 328     .node_name = "adj-midchain-tx-no-count",
 329     .runs_before = VNET_FEATURES ("interface-output"),
 330     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6],
 331 };
 332 VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = {
 333     .arc_name = "mpls-output",
 334     .node_name = "adj-midchain-tx",
 335     .runs_before = VNET_FEATURES ("interface-output"),
 336     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS],
 337 };
 338 VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = {
 339     .arc_name = "mpls-output",
 340     .node_name = "adj-midchain-tx-no-count",
 341     .runs_before = VNET_FEATURES ("interface-output"),
 342     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS],
 343 };
 344 VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = {
 345     .arc_name = "ethernet-output",
 346     .node_name = "adj-midchain-tx",
 347     .runs_before = VNET_FEATURES ("error-drop"),
 348     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET],
 349 };
 350 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = {
 351     .arc_name = "ethernet-output",
 352     .node_name = "adj-midchain-tx-no-count",
 353     .runs_before = VNET_FEATURES ("error-drop"),
 354     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET],
 355 };
 356 VNET_FEATURE_INIT (adj_midchain_tx_nsh, static) = {
 357     .arc_name = "nsh-output",
 358     .node_name = "adj-midchain-tx",
 359     .runs_before = VNET_FEATURES ("error-drop"),
 360     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_NSH],
 361 };
 362 VNET_FEATURE_INIT (adj_midchain_tx_no_count_nsh, static) = {
 363     .arc_name = "nsh-output",
 364     .node_name = "adj-midchain-tx-no-count",
 365     .runs_before = VNET_FEATURES ("error-drop"),
 366     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_NSH],
 367 };
 368
 369 static inline u32
 370 adj_get_midchain_node (vnet_link_t link)
 371 {
 372     switch (link) {
 373     case VNET_LINK_IP4:
 374         return (ip4_midchain_node.index);
 375     case VNET_LINK_IP6:
 376         return (ip6_midchain_node.index);
 377     case VNET_LINK_MPLS:
 378         return (mpls_midchain_node.index);
 379     case VNET_LINK_ETHERNET:
 380         return (adj_l2_midchain_node.index);
 381     case VNET_LINK_NSH:
 382         return (adj_nsh_midchain_node.index);
 383     case VNET_LINK_ARP:
 384         break;
 385     }
 386     ASSERT(0);
 387     return (0);
 388 }
 389
 390 static u8
 391 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
 392 {
 393     u8 arc = (u8) ~0;
 394     switch (adj->ia_link)
 395     {
 396     case VNET_LINK_IP4:
 397         {
 398             arc = ip4_main.lookup_main.output_feature_arc_index;
 399             break;
 400         }
 401     case VNET_LINK_IP6:
 402         {
 403             arc = ip6_main.lookup_main.output_feature_arc_index;
 404             break;
 405         }
 406     case VNET_LINK_MPLS:
 407         {
 408             arc = mpls_main.output_feature_arc_index;
 409             break;
 410         }
 411     case VNET_LINK_ETHERNET:
 412         {
 413             arc = ethernet_main.output_feature_arc_index;
 414             break;
 415         }
 416     case VNET_LINK_NSH:
 417         {
 418           arc = nsh_main_dummy.output_feature_arc_index;
 419           break;
 420         }
 421     case VNET_LINK_ARP:
 422         ASSERT(0);
 423         break;
 424     }
 425
 426     ASSERT (arc != (u8) ~0);
 427
 428     return (arc);
 429 }
 430
 431 static u32
 432 adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj)
 433 {
 434     return ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
 435             adj_midchain_tx_no_count_node.index :
 436             adj_midchain_tx_node.index);
 437 }
 438
 439 static u32
 440 adj_nbr_midchain_get_feature_node (ip_adjacency_t *adj)
 441 {
 442     if (adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT)
 443     {
 444         return (adj_midchain_tx_no_count_feature_node[adj->ia_link]);
 445     }
 446
 447     return (adj_midchain_tx_feature_node[adj->ia_link]);
 448 }
 449
 450 /**
 451  * adj_midchain_setup
 452  *
 453  * Setup the adj as a mid-chain
 454  */
 455 void
 456 adj_midchain_teardown (ip_adjacency_t *adj)
 457 {
 458     u32 feature_index;
 459     u8 arc_index;
 460
 461     dpo_reset(&adj->sub_type.midchain.next_dpo);
 462
 463     arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
 464     feature_index = adj_nbr_midchain_get_feature_node(adj);
 465
 466     if (0 == --adj_midchain_feat_count_per_sw_if_index[adj->ia_link][adj->rewrite_header.sw_if_index])
 467     {
 468         vnet_feature_enable_disable_with_index (arc_index, feature_index,
 469                                                 adj->rewrite_header.sw_if_index,
 470                                                 0, 0, 0);
 471     }
 472 }
 473
 474 /**
 475  * adj_midchain_setup
 476  *
 477  * Setup the adj as a mid-chain
 478  */
 479 void
 480 adj_midchain_setup (adj_index_t adj_index,
 481                     adj_midchain_fixup_t fixup,
 482                     const void *data,
 483                     adj_flags_t flags)
 484 {
 485     u32 feature_index, tx_node;
 486     ip_adjacency_t *adj;
 487     u8 arc_index;
 488
 489     ASSERT(ADJ_INDEX_INVALID != adj_index);
 490
 491     adj = adj_get(adj_index);
 492
 493     adj->sub_type.midchain.fixup_func = fixup;
 494     adj->sub_type.midchain.fixup_data = data;
 495     adj->sub_type.midchain.fei = FIB_NODE_INDEX_INVALID;
 496     adj->ia_flags |= flags;
 497
 498     arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
 499     feature_index = adj_nbr_midchain_get_feature_node(adj);
 500     tx_node = adj_nbr_midchain_get_tx_node(adj);
 501
 502     vec_validate (adj_midchain_feat_count_per_sw_if_index[adj->ia_link],
 503                   adj->rewrite_header.sw_if_index);
 504
 505     if (0 == adj_midchain_feat_count_per_sw_if_index[adj->ia_link][adj->rewrite_header.sw_if_index]++)
 506     {
 507         vnet_feature_enable_disable_with_index (arc_index, feature_index,
 508                                                 adj->rewrite_header.sw_if_index,
 509                                                 1 /* enable */, 0, 0);
 510     }
 511
 512     /*
 513      * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
 514      * The graph arc used/created here is from the midchain-tx node to the
 515      * child's registered node. This is because post adj processing the next
 516      * node are any output features, then the midchain-tx.  from there we
 517      * need to get to the stacked child's node.
 518      */
 519     dpo_stack_from_node(tx_node,
 520                         &adj->sub_type.midchain.next_dpo,
 521                         drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 522 }
 523
 524 /**
 525  * adj_nbr_midchain_update_rewrite
 526  *
 527  * Update the adjacency's rewrite string. A NULL string implies the
 528  * rewrite is reset (i.e. when ARP/ND entry is gone).
 529  * NB: the adj being updated may be handling traffic in the DP.
 530  */
 531 void
 532 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
 533                                  adj_midchain_fixup_t fixup,
 534                                  const void *fixup_data,
 535                                  adj_flags_t flags,
 536                                  u8 *rewrite)
 537 {
 538     ip_adjacency_t *adj;
 539
 540     ASSERT(ADJ_INDEX_INVALID != adj_index);
 541
 542     adj = adj_get(adj_index);
 543
 544     /*
 545      * one time only update. since we don't support changing the tunnel
 546      * src,dst, this is all we need.
 547      */
 548     if (adj->lookup_next_index != IP_LOOKUP_NEXT_MIDCHAIN &&
 549         adj->lookup_next_index != IP_LOOKUP_NEXT_MCAST_MIDCHAIN)
 550     {
 551         adj_midchain_setup(adj_index, fixup, fixup_data, flags);
 552     }
 553
 554     /*
 555      * update the rewrite with the workers paused.
 556      */
 557     adj_nbr_update_rewrite_internal(adj,
 558                                     IP_LOOKUP_NEXT_MIDCHAIN,
 559                                     adj_get_midchain_node(adj->ia_link),
 560                                     adj_nbr_midchain_get_tx_node(adj),
 561                                     rewrite);
 562 }
 563
 564 /**
 565  * adj_nbr_midchain_unstack
 566  *
 567  * Unstack the adj. stack it on drop
 568  */
 569 void
 570 adj_nbr_midchain_unstack (adj_index_t adj_index)
 571 {
 572     fib_node_index_t *entry_indicies, tmp;
 573     ip_adjacency_t *adj;
 574
 575     ASSERT(ADJ_INDEX_INVALID != adj_index);
 576     adj = adj_get (adj_index);
 577
 578     /*
 579      * check to see if this unstacking breaks a recursion loop
 580      */
 581     entry_indicies = NULL;
 582     tmp = adj->sub_type.midchain.fei;
 583     adj->sub_type.midchain.fei = FIB_NODE_INDEX_INVALID;
 584
 585     if (FIB_NODE_INDEX_INVALID != tmp)
 586     {
 587         fib_entry_recursive_loop_detect(tmp, &entry_indicies);
 588         vec_free(entry_indicies);
 589     }
 590
 591     /*
 592      * stack on the drop
 593      */
 594     dpo_stack(DPO_ADJACENCY_MIDCHAIN,
 595               vnet_link_to_dpo_proto(adj->ia_link),
 596               &adj->sub_type.midchain.next_dpo,
 597               drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 598     CLIB_MEMORY_BARRIER();
 599 }
 600
 601 void
 602 adj_nbr_midchain_stack_on_fib_entry (adj_index_t ai,
 603                                      fib_node_index_t fei,
 604                                      fib_forward_chain_type_t fct)
 605 {
 606     fib_node_index_t *entry_indicies;
 607     dpo_id_t tmp = DPO_INVALID;
 608     ip_adjacency_t *adj;
 609
 610     adj = adj_get (ai);
 611
 612     /*
 613      * check to see if this stacking will form a recursion loop
 614      */
 615     entry_indicies = NULL;
 616     adj->sub_type.midchain.fei = fei;
 617
 618     if (fib_entry_recursive_loop_detect(adj->sub_type.midchain.fei, &entry_indicies))
 619     {
 620         /*
 621          * loop formed, stack on the drop.
 622          */
 623         dpo_copy(&tmp, drop_dpo_get(fib_forw_chain_type_to_dpo_proto(fct)));
 624     }
 625     else
 626     {
 627         fib_entry_contribute_forwarding (fei, fct, &tmp);
 628
 629         if ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_IP_STACK) &&
 630             (DPO_LOAD_BALANCE == tmp.dpoi_type))
 631         {
 632             /*
 633              * do that hash now and stack on the choice.
 634              * If the choice is an incomplete adj then we will need a poke when
 635              * it becomes complete. This happens since the adj update walk propagates
 636              * as far a recursive paths.
 637              */
 638             const dpo_id_t *choice;
 639             load_balance_t *lb;
 640             int hash;
 641
 642             lb = load_balance_get (tmp.dpoi_index);
 643
 644             if (FIB_FORW_CHAIN_TYPE_UNICAST_IP4 == fct)
 645             {
 646                 hash = ip4_compute_flow_hash ((ip4_header_t *) adj_get_rewrite (ai),
 647                                               lb->lb_hash_config);
 648             }
 649             else if (FIB_FORW_CHAIN_TYPE_UNICAST_IP6 == fct)
 650             {
 651                 hash = ip6_compute_flow_hash ((ip6_header_t *) adj_get_rewrite (ai),
 652                                               lb->lb_hash_config);
 653             }
 654             else
 655             {
 656                 hash = 0;
 657                 ASSERT(0);
 658             }
 659
 660             choice = load_balance_get_bucket_i (lb, hash & lb->lb_n_buckets_minus_1);
 661             dpo_copy (&tmp, choice);
 662         }
 663     }
 664     adj_nbr_midchain_stack (ai, &tmp);
 665     dpo_reset(&tmp);
 666     vec_free(entry_indicies);
 667 }
 668
 669 /**
 670  * adj_nbr_midchain_stack
 671  */
 672 void
 673 adj_nbr_midchain_stack (adj_index_t adj_index,
 674                         const dpo_id_t *next)
 675 {
 676     ip_adjacency_t *adj;
 677
 678     ASSERT(ADJ_INDEX_INVALID != adj_index);
 679
 680     adj = adj_get(adj_index);
 681
 682     ASSERT((IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index) ||
 683            (IP_LOOKUP_NEXT_MCAST_MIDCHAIN == adj->lookup_next_index));
 684
 685     dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj),
 686                         &adj->sub_type.midchain.next_dpo,
 687                         next);
 688 }
 689
 690 int
 691 adj_ndr_midchain_recursive_loop_detect (adj_index_t ai,
 692                                         fib_node_index_t **entry_indicies)
 693 {
 694     fib_node_index_t *entry_index, *entries;
 695     ip_adjacency_t * adj;
 696
 697     adj = adj_get(ai);
 698     entries = *entry_indicies;
 699
 700     vec_foreach(entry_index, entries)
 701     {
 702         if (*entry_index == adj->sub_type.midchain.fei)
 703         {
 704             /*
 705              * The entry this midchain links to is already in the set
 706              * of visited entries, this is a loop
 707              */
 708             adj->ia_flags |= ADJ_FLAG_MIDCHAIN_LOOPED;
 709             return (1);
 710         }
 711     }
 712
 713     adj->ia_flags &= ~ADJ_FLAG_MIDCHAIN_LOOPED;
 714     return (0);
 715 }
 716
 717 u8*
 718 format_adj_midchain (u8* s, va_list *ap)
 719 {
 720     index_t index = va_arg(*ap, index_t);
 721     u32 indent = va_arg(*ap, u32);
 722     ip_adjacency_t * adj = adj_get(index);
 723
 724     s = format (s, "%U", format_vnet_link, adj->ia_link);
 725     if (adj->rewrite_header.flags & VNET_REWRITE_HAS_FEATURES)
 726         s = format(s, " [features]");
 727     s = format (s, " via %U",
 728                 format_ip46_address, &adj->sub_type.nbr.next_hop,
 729                 adj_proto_to_46(adj->ia_nh_proto));
 730     s = format (s, " %U",
 731                 format_vnet_rewrite,
 732                 &adj->rewrite_header, sizeof (adj->rewrite_data), indent);
 733     s = format (s, "\n%Ustacked-on",
 734                 format_white_space, indent);
 735
 736     if (FIB_NODE_INDEX_INVALID != adj->sub_type.midchain.fei)
 737     {
 738         s = format (s, " entry:%d", adj->sub_type.midchain.fei);
 739
 740     }
 741     s = format (s, ":\n%U%U",
 742                 format_white_space, indent+2,
 743                 format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
 744
 745     return (s);
 746 }
 747
 748 static void
 749 adj_dpo_lock (dpo_id_t *dpo)
 750 {
 751     adj_lock(dpo->dpoi_index);
 752 }
 753 static void
 754 adj_dpo_unlock (dpo_id_t *dpo)
 755 {
 756     adj_unlock(dpo->dpoi_index);
 757 }
 758
 759 const static dpo_vft_t adj_midchain_dpo_vft = {
 760     .dv_lock = adj_dpo_lock,
 761     .dv_unlock = adj_dpo_unlock,
 762     .dv_format = format_adj_midchain,
 763     .dv_get_urpf = adj_dpo_get_urpf,
 764 };
 765
 766 /**
 767  * @brief The per-protocol VLIB graph nodes that are assigned to a midchain
 768  *        object.
 769  *
 770  * this means that these graph nodes are ones from which a midchain is the
 771  * parent object in the DPO-graph.
 772  */
 773 const static char* const midchain_ip4_nodes[] =
 774 {
 775     "ip4-midchain",
 776     NULL,
 777 };
 778 const static char* const midchain_ip6_nodes[] =
 779 {
 780     "ip6-midchain",
 781     NULL,
 782 };
 783 const static char* const midchain_mpls_nodes[] =
 784 {
 785     "mpls-midchain",
 786     NULL,
 787 };
 788 const static char* const midchain_ethernet_nodes[] =
 789 {
 790     "adj-l2-midchain",
 791     NULL,
 792 };
 793 const static char* const midchain_nsh_nodes[] =
 794 {
 795     "adj-nsh-midchain",
 796     NULL,
 797 };
 798
 799 const static char* const * const midchain_nodes[DPO_PROTO_NUM] =
 800 {
 801     [DPO_PROTO_IP4]  = midchain_ip4_nodes,
 802     [DPO_PROTO_IP6]  = midchain_ip6_nodes,
 803     [DPO_PROTO_MPLS] = midchain_mpls_nodes,
 804     [DPO_PROTO_ETHERNET] = midchain_ethernet_nodes,
 805     [DPO_PROTO_NSH] = midchain_nsh_nodes,
 806 };
 807
 808 void
 809 adj_midchain_module_init (void)
 810 {
 811     dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes);
 812 }