src/vnet/adj/adj_midchain.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/adj/adj_nbr.h>
  17 #include <vnet/adj/adj_internal.h>
  18 #include <vnet/adj/adj_l2.h>
  19 #include <vnet/adj/adj_nsh.h>
  20 #include <vnet/adj/adj_midchain.h>
  21 #include <vnet/ethernet/arp_packet.h>
  22 #include <vnet/dpo/drop_dpo.h>
  23 #include <vnet/fib/fib_walk.h>
  24
  25 /**
  26  * The two midchain tx feature node indices
  27  */
  28 static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM];
  29 static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM];
  30
  31 /**
  32  * @brief Trace data for packets traversing the midchain tx node
  33  */
  34 typedef struct adj_midchain_tx_trace_t_
  35 {
  36     /**
  37      * @brief the midchain adj we are traversing
  38      */
  39     adj_index_t ai;
  40 } adj_midchain_tx_trace_t;
  41
  42 always_inline uword
  43 adj_midchain_tx_inline (vlib_main_t * vm,
  44                         vlib_node_runtime_t * node,
  45                         vlib_frame_t * frame,
  46                         int interface_count)
  47 {
  48     u32 * from, * to_next, n_left_from, n_left_to_next;
  49     u32 next_index;
  50     vnet_main_t *vnm = vnet_get_main ();
  51     vnet_interface_main_t *im = &vnm->interface_main;
  52     u32 cpu_index = vm->cpu_index;
  53
  54     /* Vector of buffer / pkt indices we're supposed to process */
  55     from = vlib_frame_vector_args (frame);
  56
  57     /* Number of buffers / pkts */
  58     n_left_from = frame->n_vectors;
  59
  60     /* Speculatively send the first buffer to the last disposition we used */
  61     next_index = node->cached_next_index;
  62
  63     while (n_left_from > 0)
  64     {
  65         /* set up to enqueue to our disposition with index = next_index */
  66         vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
  67
  68
  69         while (n_left_from >= 4 && n_left_to_next > 2)
  70         {
  71             u32 bi0, adj_index0, next0;
  72             const ip_adjacency_t * adj0;
  73             const dpo_id_t *dpo0;
  74             vlib_buffer_t * b0;
  75             u32 bi1, adj_index1, next1;
  76             const ip_adjacency_t * adj1;
  77             const dpo_id_t *dpo1;
  78             vlib_buffer_t * b1;
  79
  80             /* Prefetch next iteration. */
  81             {
  82                 vlib_buffer_t * p2, * p3;
  83
  84                 p2 = vlib_get_buffer (vm, from[2]);
  85                 p3 = vlib_get_buffer (vm, from[3]);
  86
  87                 vlib_prefetch_buffer_header (p2, LOAD);
  88                 vlib_prefetch_buffer_header (p3, LOAD);
  89
  90                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
  91                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
  92             }
  93
  94             bi0 = from[0];
  95             to_next[0] = bi0;
  96             bi1 = from[1];
  97             to_next[1] = bi1;
  98
  99             from += 2;
 100             to_next += 2;
 101             n_left_from -= 2;
 102             n_left_to_next -= 2;
 103
 104             b0 = vlib_get_buffer(vm, bi0);
 105             b1 = vlib_get_buffer(vm, bi1);
 106
 107             /* Follow the DPO on which the midchain is stacked */
 108             adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 109             adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 110
 111             adj0 = adj_get(adj_index0);
 112             adj1 = adj_get(adj_index1);
 113
 114             dpo0 = &adj0->sub_type.midchain.next_dpo;
 115             dpo1 = &adj1->sub_type.midchain.next_dpo;
 116
 117             next0 = dpo0->dpoi_next_node;
 118             next1 = dpo1->dpoi_next_node;
 119
 120             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 121             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 122
 123             if (interface_count)
 124             {
 125                 vlib_increment_combined_counter (im->combined_sw_if_counters
 126                                                  + VNET_INTERFACE_COUNTER_TX,
 127                                                  cpu_index,
 128                                                  adj0->rewrite_header.sw_if_index,
 129                                                  1,
 130                                                  vlib_buffer_length_in_chain (vm, b0));
 131                 vlib_increment_combined_counter (im->combined_sw_if_counters
 132                                                  + VNET_INTERFACE_COUNTER_TX,
 133                                                  cpu_index,
 134                                                  adj1->rewrite_header.sw_if_index,
 135                                                  1,
 136                                                  vlib_buffer_length_in_chain (vm, b1));
 137             }
 138
 139             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 140             {
 141                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 142                                                               b0, sizeof (*tr));
 143                 tr->ai = adj_index0;
 144             }
 145             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 146             {
 147                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 148                                                               b1, sizeof (*tr));
 149                 tr->ai = adj_index1;
 150             }
 151
 152             vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 153                                              to_next, n_left_to_next,
 154                                              bi0, bi1,
 155                                              next0, next1);
 156         }
 157         while (n_left_from > 0 && n_left_to_next > 0)
 158         {
 159             u32 bi0, adj_index0, next0;
 160             const ip_adjacency_t * adj0;
 161             const dpo_id_t *dpo0;
 162             vlib_buffer_t * b0;
 163
 164             bi0 = from[0];
 165             to_next[0] = bi0;
 166             from += 1;
 167             to_next += 1;
 168             n_left_from -= 1;
 169             n_left_to_next -= 1;
 170
 171             b0 = vlib_get_buffer(vm, bi0);
 172
 173             /* Follow the DPO on which the midchain is stacked */
 174             adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 175             adj0 = adj_get(adj_index0);
 176             dpo0 = &adj0->sub_type.midchain.next_dpo;
 177             next0 = dpo0->dpoi_next_node;
 178             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 179
 180             if (interface_count)
 181             {
 182                 vlib_increment_combined_counter (im->combined_sw_if_counters
 183                                                  + VNET_INTERFACE_COUNTER_TX,
 184                                                  cpu_index,
 185                                                  adj0->rewrite_header.sw_if_index,
 186                                                  1,
 187                                                  vlib_buffer_length_in_chain (vm, b0));
 188             }
 189
 190             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 191             {
 192                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 193                                                               b0, sizeof (*tr));
 194                 tr->ai = adj_index0;
 195             }
 196
 197             vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 198                                              to_next, n_left_to_next,
 199                                              bi0, next0);
 200         }
 201
 202         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 203     }
 204
 205     return frame->n_vectors;
 206 }
 207
 208 static u8 *
 209 format_adj_midchain_tx_trace (u8 * s, va_list * args)
 210 {
 211     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 212     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 213     adj_midchain_tx_trace_t *tr = va_arg (*args, adj_midchain_tx_trace_t*);
 214
 215     s = format(s, "adj-midchain:[%d]:%U", tr->ai,
 216                format_ip_adjacency, tr->ai,
 217                FORMAT_IP_ADJACENCY_NONE);
 218
 219     return (s);
 220 }
 221
 222 static uword
 223 adj_midchain_tx (vlib_main_t * vm,
 224                  vlib_node_runtime_t * node,
 225                  vlib_frame_t * frame)
 226 {
 227     return (adj_midchain_tx_inline(vm, node, frame, 1));
 228 }
 229
 230 VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = {
 231     .function = adj_midchain_tx,
 232     .name = "adj-midchain-tx",
 233     .vector_size = sizeof (u32),
 234
 235     .format_trace = format_adj_midchain_tx_trace,
 236
 237     .n_next_nodes = 1,
 238     .next_nodes = {
 239         [0] = "error-drop",
 240     },
 241 };
 242
 243 static uword
 244 adj_midchain_tx_no_count (vlib_main_t * vm,
 245                           vlib_node_runtime_t * node,
 246                           vlib_frame_t * frame)
 247 {
 248     return (adj_midchain_tx_inline(vm, node, frame, 0));
 249 }
 250
 251 VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = {
 252     .function = adj_midchain_tx_no_count,
 253     .name = "adj-midchain-tx-no-count",
 254     .vector_size = sizeof (u32),
 255
 256     .format_trace = format_adj_midchain_tx_trace,
 257
 258     .n_next_nodes = 1,
 259     .next_nodes = {
 260         [0] = "error-drop",
 261     },
 262 };
 263
 264 VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = {
 265     .arc_name = "ip4-output",
 266     .node_name = "adj-midchain-tx",
 267     .runs_before = VNET_FEATURES ("interface-output"),
 268     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4],
 269 };
 270 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = {
 271     .arc_name = "ip4-output",
 272     .node_name = "adj-midchain-tx-no-count",
 273     .runs_before = VNET_FEATURES ("interface-output"),
 274     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4],
 275 };
 276 VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = {
 277     .arc_name = "ip6-output",
 278     .node_name = "adj-midchain-tx",
 279     .runs_before = VNET_FEATURES ("interface-output"),
 280     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6],
 281 };
 282 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = {
 283     .arc_name = "ip6-output",
 284     .node_name = "adj-midchain-tx-no-count",
 285     .runs_before = VNET_FEATURES ("interface-output"),
 286     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6],
 287 };
 288 VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = {
 289     .arc_name = "mpls-output",
 290     .node_name = "adj-midchain-tx",
 291     .runs_before = VNET_FEATURES ("interface-output"),
 292     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS],
 293 };
 294 VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = {
 295     .arc_name = "mpls-output",
 296     .node_name = "adj-midchain-tx-no-count",
 297     .runs_before = VNET_FEATURES ("interface-output"),
 298     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS],
 299 };
 300 VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = {
 301     .arc_name = "ethernet-output",
 302     .node_name = "adj-midchain-tx",
 303     .runs_before = VNET_FEATURES ("error-drop"),
 304     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET],
 305 };
 306 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = {
 307     .arc_name = "ethernet-output",
 308     .node_name = "adj-midchain-tx-no-count",
 309     .runs_before = VNET_FEATURES ("error-drop"),
 310     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET],
 311 };
 312 VNET_FEATURE_INIT (adj_midchain_tx_nsh, static) = {
 313     .arc_name = "nsh-output",
 314     .node_name = "adj-midchain-tx",
 315     .runs_before = VNET_FEATURES ("error-drop"),
 316     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_NSH],
 317 };
 318 VNET_FEATURE_INIT (adj_midchain_tx_no_count_nsh, static) = {
 319     .arc_name = "nsh-output",
 320     .node_name = "adj-midchain-tx-no-count",
 321     .runs_before = VNET_FEATURES ("error-drop"),
 322     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_NSH],
 323 };
 324
 325 static inline u32
 326 adj_get_midchain_node (vnet_link_t link)
 327 {
 328     switch (link) {
 329     case VNET_LINK_IP4:
 330         return (ip4_midchain_node.index);
 331     case VNET_LINK_IP6:
 332         return (ip6_midchain_node.index);
 333     case VNET_LINK_MPLS:
 334         return (mpls_midchain_node.index);
 335     case VNET_LINK_ETHERNET:
 336         return (adj_l2_midchain_node.index);
 337     case VNET_LINK_NSH:
 338         return (adj_nsh_midchain_node.index);
 339     case VNET_LINK_ARP:
 340         break;
 341     }
 342     ASSERT(0);
 343     return (0);
 344 }
 345
 346 static u8
 347 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
 348 {
 349   u8 arc = (u8) ~0;
 350     switch (adj->ia_link)
 351     {
 352     case VNET_LINK_IP4:
 353         {
 354             arc = ip4_main.lookup_main.output_feature_arc_index;
 355             break;
 356         }
 357     case VNET_LINK_IP6:
 358         {
 359             arc = ip6_main.lookup_main.output_feature_arc_index;
 360             break;
 361         }
 362     case VNET_LINK_MPLS:
 363         {
 364             arc = mpls_main.output_feature_arc_index;
 365             break;
 366         }
 367     case VNET_LINK_ETHERNET:
 368         {
 369             arc = ethernet_main.output_feature_arc_index;
 370             break;
 371         }
 372     case VNET_LINK_NSH:
 373         {
 374           arc = nsh_main_dummy.output_feature_arc_index;
 375           break;
 376         }
 377     case VNET_LINK_ARP:
 378         ASSERT(0);
 379         break;
 380     }
 381
 382     ASSERT (arc != (u8) ~0);
 383
 384     return (arc);
 385 }
 386
 387 /**
 388  * adj_nbr_midchain_update_rewrite
 389  *
 390  * Update the adjacency's rewrite string. A NULL string implies the
 391  * rewrite is reset (i.e. when ARP/ND etnry is gone).
 392  * NB: the adj being updated may be handling traffic in the DP.
 393  */
 394 void
 395 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
 396                                  adj_midchain_fixup_t fixup,
 397                                  adj_midchain_flag_t flags,
 398                                  u8 *rewrite)
 399 {
 400     ip_adjacency_t *adj;
 401     u8 arc_index;
 402     u32 feature_index;
 403
 404     ASSERT(ADJ_INDEX_INVALID != adj_index);
 405
 406     adj = adj_get(adj_index);
 407
 408     /*
 409      * one time only update. since we don't support chainging the tunnel
 410      * src,dst, this is all we need.
 411      */
 412     ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP);
 413     /*
 414      * tunnels can always provide a rewrite.
 415      */
 416     ASSERT(NULL != rewrite);
 417
 418     adj->sub_type.midchain.fixup_func = fixup;
 419
 420     arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
 421     feature_index = (flags & ADJ_MIDCHAIN_FLAG_NO_COUNT) ?
 422                     adj_midchain_tx_no_count_feature_node[adj->ia_link] :
 423                     adj_midchain_tx_feature_node[adj->ia_link];
 424
 425     adj->sub_type.midchain.tx_function_node = (flags & ADJ_MIDCHAIN_FLAG_NO_COUNT) ?
 426                                                adj_midchain_tx_no_count_node.index :
 427                                                adj_midchain_tx_node.index;
 428
 429     vnet_feature_enable_disable_with_index (arc_index, feature_index,
 430                                             adj->rewrite_header.sw_if_index,
 431                                             1 /* enable */, 0, 0);
 432
 433     /*
 434      * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
 435      * The graph arc used/created here is from the midchain-tx node to the
 436      * child's registered node. This is because post adj processing the next
 437      * node are any output features, then the midchain-tx.  from there we
 438      * need to get to the stacked child's node.
 439      */
 440     dpo_stack_from_node(adj->sub_type.midchain.tx_function_node,
 441                         &adj->sub_type.midchain.next_dpo,
 442                         drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 443
 444     /*
 445      * update the rewirte with the workers paused.
 446      */
 447     adj_nbr_update_rewrite_internal(adj,
 448                                     IP_LOOKUP_NEXT_MIDCHAIN,
 449                                     adj_get_midchain_node(adj->ia_link),
 450                                     adj->sub_type.midchain.tx_function_node,
 451                                     rewrite);
 452 }
 453
 454 /**
 455  * adj_nbr_midchain_unstack
 456  *
 457  * Unstack the adj. stack it on drop
 458  */
 459 void
 460 adj_nbr_midchain_unstack (adj_index_t adj_index)
 461 {
 462     ip_adjacency_t *adj;
 463
 464     ASSERT(ADJ_INDEX_INVALID != adj_index);
 465
 466     adj = adj_get(adj_index);
 467
 468     /*
 469      * stack on the drop
 470      */
 471     dpo_stack(DPO_ADJACENCY_MIDCHAIN,
 472               vnet_link_to_dpo_proto(adj->ia_link),
 473               &adj->sub_type.midchain.next_dpo,
 474               drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 475
 476     CLIB_MEMORY_BARRIER();
 477 }
 478
 479 /**
 480  * adj_nbr_midchain_stack
 481  */
 482 void
 483 adj_nbr_midchain_stack (adj_index_t adj_index,
 484                         const dpo_id_t *next)
 485 {
 486     ip_adjacency_t *adj;
 487
 488     ASSERT(ADJ_INDEX_INVALID != adj_index);
 489
 490     adj = adj_get(adj_index);
 491
 492     ASSERT(IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index);
 493
 494     dpo_stack_from_node(adj->sub_type.midchain.tx_function_node,
 495                         &adj->sub_type.midchain.next_dpo,
 496                         next);
 497 }
 498
 499 u8*
 500 format_adj_midchain (u8* s, va_list *ap)
 501 {
 502     index_t index = va_arg(*ap, index_t);
 503     u32 indent = va_arg(*ap, u32);
 504     ip_adjacency_t * adj = adj_get(index);
 505
 506     s = format (s, "%U", format_vnet_link, adj->ia_link);
 507     s = format (s, " via %U ",
 508                 format_ip46_address, &adj->sub_type.nbr.next_hop);
 509     s = format (s, " %U",
 510                 format_vnet_rewrite,
 511                 &adj->rewrite_header, sizeof (adj->rewrite_data), indent);
 512     s = format (s, "\n%Ustacked-on:\n%U%U",
 513                 format_white_space, indent,
 514                 format_white_space, indent+2,
 515                 format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
 516
 517     return (s);
 518 }
 519
 520 static void
 521 adj_dpo_lock (dpo_id_t *dpo)
 522 {
 523     adj_lock(dpo->dpoi_index);
 524 }
 525 static void
 526 adj_dpo_unlock (dpo_id_t *dpo)
 527 {
 528     adj_unlock(dpo->dpoi_index);
 529 }
 530
 531 const static dpo_vft_t adj_midchain_dpo_vft = {
 532     .dv_lock = adj_dpo_lock,
 533     .dv_unlock = adj_dpo_unlock,
 534     .dv_format = format_adj_midchain,
 535 };
 536
 537 /**
 538  * @brief The per-protocol VLIB graph nodes that are assigned to a midchain
 539  *        object.
 540  *
 541  * this means that these graph nodes are ones from which a midchain is the
 542  * parent object in the DPO-graph.
 543  */
 544 const static char* const midchain_ip4_nodes[] =
 545 {
 546     "ip4-midchain",
 547     NULL,
 548 };
 549 const static char* const midchain_ip6_nodes[] =
 550 {
 551     "ip6-midchain",
 552     NULL,
 553 };
 554 const static char* const midchain_mpls_nodes[] =
 555 {
 556     "mpls-midchain",
 557     NULL,
 558 };
 559 const static char* const midchain_ethernet_nodes[] =
 560 {
 561     "adj-l2-midchain",
 562     NULL,
 563 };
 564 const static char* const midchain_nsh_nodes[] =
 565 {
 566     "adj-nsh-midchain",
 567     NULL,
 568 };
 569
 570 const static char* const * const midchain_nodes[DPO_PROTO_NUM] =
 571 {
 572     [DPO_PROTO_IP4]  = midchain_ip4_nodes,
 573     [DPO_PROTO_IP6]  = midchain_ip6_nodes,
 574     [DPO_PROTO_MPLS] = midchain_mpls_nodes,
 575     [DPO_PROTO_ETHERNET] = midchain_ethernet_nodes,
 576     [DPO_PROTO_NSH] = midchain_nsh_nodes,
 577 };
 578
 579 void
 580 adj_midchain_module_init (void)
 581 {
 582     dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes);
 583 }