src/vnet/adj/adj_midchain.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/adj/adj_nbr.h>
  17 #include <vnet/adj/adj_internal.h>
  18 #include <vnet/adj/adj_l2.h>
  19 #include <vnet/adj/adj_nsh.h>
  20 #include <vnet/adj/adj_midchain.h>
  21 #include <vnet/ethernet/arp_packet.h>
  22 #include <vnet/dpo/drop_dpo.h>
  23 #include <vnet/fib/fib_walk.h>
  24
  25 /**
  26  * The two midchain tx feature node indices
  27  */
  28 static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM];
  29 static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM];
  30
  31 /**
  32  * @brief Trace data for packets traversing the midchain tx node
  33  */
  34 typedef struct adj_midchain_tx_trace_t_
  35 {
  36     /**
  37      * @brief the midchain adj we are traversing
  38      */
  39     adj_index_t ai;
  40 } adj_midchain_tx_trace_t;
  41
  42 always_inline uword
  43 adj_midchain_tx_inline (vlib_main_t * vm,
  44                         vlib_node_runtime_t * node,
  45                         vlib_frame_t * frame,
  46                         int interface_count)
  47 {
  48     u32 * from, * to_next, n_left_from, n_left_to_next;
  49     u32 next_index;
  50     vnet_main_t *vnm = vnet_get_main ();
  51     vnet_interface_main_t *im = &vnm->interface_main;
  52     u32 cpu_index = vm->cpu_index;
  53
  54     /* Vector of buffer / pkt indices we're supposed to process */
  55     from = vlib_frame_vector_args (frame);
  56
  57     /* Number of buffers / pkts */
  58     n_left_from = frame->n_vectors;
  59
  60     /* Speculatively send the first buffer to the last disposition we used */
  61     next_index = node->cached_next_index;
  62
  63     while (n_left_from > 0)
  64     {
  65         /* set up to enqueue to our disposition with index = next_index */
  66         vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
  67
  68
  69         while (n_left_from >= 4 && n_left_to_next > 2)
  70         {
  71             u32 bi0, adj_index0, next0;
  72             const ip_adjacency_t * adj0;
  73             const dpo_id_t *dpo0;
  74             vlib_buffer_t * b0;
  75             u32 bi1, adj_index1, next1;
  76             const ip_adjacency_t * adj1;
  77             const dpo_id_t *dpo1;
  78             vlib_buffer_t * b1;
  79
  80             /* Prefetch next iteration. */
  81             {
  82                 vlib_buffer_t * p2, * p3;
  83
  84                 p2 = vlib_get_buffer (vm, from[2]);
  85                 p3 = vlib_get_buffer (vm, from[3]);
  86
  87                 vlib_prefetch_buffer_header (p2, LOAD);
  88                 vlib_prefetch_buffer_header (p3, LOAD);
  89
  90                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
  91                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
  92             }
  93
  94             bi0 = from[0];
  95             to_next[0] = bi0;
  96             bi1 = from[1];
  97             to_next[1] = bi1;
  98
  99             from += 2;
 100             to_next += 2;
 101             n_left_from -= 2;
 102             n_left_to_next -= 2;
 103
 104             b0 = vlib_get_buffer(vm, bi0);
 105             b1 = vlib_get_buffer(vm, bi1);
 106
 107             /* Follow the DPO on which the midchain is stacked */
 108             adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 109             adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 110
 111             adj0 = adj_get(adj_index0);
 112             adj1 = adj_get(adj_index1);
 113
 114             dpo0 = &adj0->sub_type.midchain.next_dpo;
 115             dpo1 = &adj1->sub_type.midchain.next_dpo;
 116
 117             next0 = dpo0->dpoi_next_node;
 118             next1 = dpo1->dpoi_next_node;
 119
 120             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 121             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 122
 123             if (interface_count)
 124             {
 125                 vlib_increment_combined_counter (im->combined_sw_if_counters
 126                                                  + VNET_INTERFACE_COUNTER_TX,
 127                                                  cpu_index,
 128                                                  adj0->rewrite_header.sw_if_index,
 129                                                  1,
 130                                                  vlib_buffer_length_in_chain (vm, b0));
 131                 vlib_increment_combined_counter (im->combined_sw_if_counters
 132                                                  + VNET_INTERFACE_COUNTER_TX,
 133                                                  cpu_index,
 134                                                  adj1->rewrite_header.sw_if_index,
 135                                                  1,
 136                                                  vlib_buffer_length_in_chain (vm, b1));
 137             }
 138
 139             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 140             {
 141                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 142                                                               b0, sizeof (*tr));
 143                 tr->ai = adj_index0;
 144             }
 145             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 146             {
 147                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 148                                                               b1, sizeof (*tr));
 149                 tr->ai = adj_index1;
 150             }
 151
 152             vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 153                                              to_next, n_left_to_next,
 154                                              bi0, bi1,
 155                                              next0, next1);
 156         }
 157         while (n_left_from > 0 && n_left_to_next > 0)
 158         {
 159             u32 bi0, adj_index0, next0;
 160             const ip_adjacency_t * adj0;
 161             const dpo_id_t *dpo0;
 162             vlib_buffer_t * b0;
 163
 164             bi0 = from[0];
 165             to_next[0] = bi0;
 166             from += 1;
 167             to_next += 1;
 168             n_left_from -= 1;
 169             n_left_to_next -= 1;
 170
 171             b0 = vlib_get_buffer(vm, bi0);
 172
 173             /* Follow the DPO on which the midchain is stacked */
 174             adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 175             adj0 = adj_get(adj_index0);
 176             dpo0 = &adj0->sub_type.midchain.next_dpo;
 177             next0 = dpo0->dpoi_next_node;
 178             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 179
 180             if (interface_count)
 181             {
 182                 vlib_increment_combined_counter (im->combined_sw_if_counters
 183                                                  + VNET_INTERFACE_COUNTER_TX,
 184                                                  cpu_index,
 185                                                  adj0->rewrite_header.sw_if_index,
 186                                                  1,
 187                                                  vlib_buffer_length_in_chain (vm, b0));
 188             }
 189
 190             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 191             {
 192                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 193                                                               b0, sizeof (*tr));
 194                 tr->ai = adj_index0;
 195             }
 196
 197             vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 198                                              to_next, n_left_to_next,
 199                                              bi0, next0);
 200         }
 201
 202         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 203     }
 204
 205     return frame->n_vectors;
 206 }
 207
 208 static u8 *
 209 format_adj_midchain_tx_trace (u8 * s, va_list * args)
 210 {
 211     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 212     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 213     adj_midchain_tx_trace_t *tr = va_arg (*args, adj_midchain_tx_trace_t*);
 214
 215     s = format(s, "adj-midchain:[%d]:%U", tr->ai,
 216                format_ip_adjacency, tr->ai,
 217                FORMAT_IP_ADJACENCY_NONE);
 218
 219     return (s);
 220 }
 221
 222 static uword
 223 adj_midchain_tx (vlib_main_t * vm,
 224                  vlib_node_runtime_t * node,
 225                  vlib_frame_t * frame)
 226 {
 227     return (adj_midchain_tx_inline(vm, node, frame, 1));
 228 }
 229
 230 VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = {
 231     .function = adj_midchain_tx,
 232     .name = "adj-midchain-tx",
 233     .vector_size = sizeof (u32),
 234
 235     .format_trace = format_adj_midchain_tx_trace,
 236
 237     .n_next_nodes = 1,
 238     .next_nodes = {
 239         [0] = "error-drop",
 240     },
 241 };
 242
 243 static uword
 244 adj_midchain_tx_no_count (vlib_main_t * vm,
 245                           vlib_node_runtime_t * node,
 246                           vlib_frame_t * frame)
 247 {
 248     return (adj_midchain_tx_inline(vm, node, frame, 0));
 249 }
 250
 251 VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = {
 252     .function = adj_midchain_tx_no_count,
 253     .name = "adj-midchain-tx-no-count",
 254     .vector_size = sizeof (u32),
 255
 256     .format_trace = format_adj_midchain_tx_trace,
 257
 258     .n_next_nodes = 1,
 259     .next_nodes = {
 260         [0] = "error-drop",
 261     },
 262 };
 263
 264 VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = {
 265     .arc_name = "ip4-output",
 266     .node_name = "adj-midchain-tx",
 267     .runs_before = VNET_FEATURES ("interface-output"),
 268     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4],
 269 };
 270 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = {
 271     .arc_name = "ip4-output",
 272     .node_name = "adj-midchain-tx-no-count",
 273     .runs_before = VNET_FEATURES ("interface-output"),
 274     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4],
 275 };
 276 VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = {
 277     .arc_name = "ip6-output",
 278     .node_name = "adj-midchain-tx",
 279     .runs_before = VNET_FEATURES ("interface-output"),
 280     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6],
 281 };
 282 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = {
 283     .arc_name = "ip6-output",
 284     .node_name = "adj-midchain-tx-no-count",
 285     .runs_before = VNET_FEATURES ("interface-output"),
 286     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6],
 287 };
 288 VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = {
 289     .arc_name = "mpls-output",
 290     .node_name = "adj-midchain-tx",
 291     .runs_before = VNET_FEATURES ("interface-output"),
 292     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS],
 293 };
 294 VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = {
 295     .arc_name = "mpls-output",
 296     .node_name = "adj-midchain-tx-no-count",
 297     .runs_before = VNET_FEATURES ("interface-output"),
 298     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS],
 299 };
 300 VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = {
 301     .arc_name = "ethernet-output",
 302     .node_name = "adj-midchain-tx",
 303     .runs_before = VNET_FEATURES ("error-drop"),
 304     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET],
 305 };
 306 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = {
 307     .arc_name = "ethernet-output",
 308     .node_name = "adj-midchain-tx-no-count",
 309     .runs_before = VNET_FEATURES ("error-drop"),
 310     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET],
 311 };
 312 VNET_FEATURE_INIT (adj_midchain_tx_nsh, static) = {
 313     .arc_name = "nsh-output",
 314     .node_name = "adj-midchain-tx",
 315     .runs_before = VNET_FEATURES ("error-drop"),
 316     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_NSH],
 317 };
 318 VNET_FEATURE_INIT (adj_midchain_tx_no_count_nsh, static) = {
 319     .arc_name = "nsh-output",
 320     .node_name = "adj-midchain-tx-no-count",
 321     .runs_before = VNET_FEATURES ("error-drop"),
 322     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_NSH],
 323 };
 324
 325 static inline u32
 326 adj_get_midchain_node (vnet_link_t link)
 327 {
 328     switch (link) {
 329     case VNET_LINK_IP4:
 330         return (ip4_midchain_node.index);
 331     case VNET_LINK_IP6:
 332         return (ip6_midchain_node.index);
 333     case VNET_LINK_MPLS:
 334         return (mpls_midchain_node.index);
 335     case VNET_LINK_ETHERNET:
 336         return (adj_l2_midchain_node.index);
 337     case VNET_LINK_NSH:
 338         return (adj_nsh_midchain_node.index);
 339     case VNET_LINK_ARP:
 340         break;
 341     }
 342     ASSERT(0);
 343     return (0);
 344 }
 345
 346 static u8
 347 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
 348 {
 349   u8 arc = (u8) ~0;
 350     switch (adj->ia_link)
 351     {
 352     case VNET_LINK_IP4:
 353         {
 354             arc = ip4_main.lookup_main.output_feature_arc_index;
 355             break;
 356         }
 357     case VNET_LINK_IP6:
 358         {
 359             arc = ip6_main.lookup_main.output_feature_arc_index;
 360             break;
 361         }
 362     case VNET_LINK_MPLS:
 363         {
 364             arc = mpls_main.output_feature_arc_index;
 365             break;
 366         }
 367     case VNET_LINK_ETHERNET:
 368         {
 369             arc = ethernet_main.output_feature_arc_index;
 370             break;
 371         }
 372     case VNET_LINK_NSH:
 373         {
 374           arc = nsh_main_dummy.output_feature_arc_index;
 375           break;
 376         }
 377     case VNET_LINK_ARP:
 378         ASSERT(0);
 379         break;
 380     }
 381
 382     ASSERT (arc != (u8) ~0);
 383
 384     return (arc);
 385 }
 386
 387 static u32
 388 adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj)
 389 {
 390     return ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
 391             adj_midchain_tx_no_count_node.index :
 392             adj_midchain_tx_node.index);
 393 }
 394
 395 /**
 396  * adj_nbr_midchain_update_rewrite
 397  *
 398  * Update the adjacency's rewrite string. A NULL string implies the
 399  * rewrite is reset (i.e. when ARP/ND etnry is gone).
 400  * NB: the adj being updated may be handling traffic in the DP.
 401  */
 402 void
 403 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
 404                                  adj_midchain_fixup_t fixup,
 405                                  adj_flags_t flags,
 406                                  u8 *rewrite)
 407 {
 408     u32 feature_index, tx_node;
 409     ip_adjacency_t *adj;
 410     u8 arc_index;
 411
 412     ASSERT(ADJ_INDEX_INVALID != adj_index);
 413
 414     adj = adj_get(adj_index);
 415
 416     /*
 417      * one time only update. since we don't support chainging the tunnel
 418      * src,dst, this is all we need.
 419      */
 420     ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP);
 421     /*
 422      * tunnels can always provide a rewrite.
 423      */
 424     ASSERT(NULL != rewrite);
 425
 426     adj->sub_type.midchain.fixup_func = fixup;
 427     adj->ia_flags |= flags;
 428
 429     arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
 430     feature_index = (flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
 431                     adj_midchain_tx_no_count_feature_node[adj->ia_link] :
 432                     adj_midchain_tx_feature_node[adj->ia_link];
 433
 434     tx_node = adj_nbr_midchain_get_tx_node(adj);
 435
 436     vnet_feature_enable_disable_with_index (arc_index, feature_index,
 437                                             adj->rewrite_header.sw_if_index,
 438                                             1 /* enable */, 0, 0);
 439
 440     /*
 441      * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
 442      * The graph arc used/created here is from the midchain-tx node to the
 443      * child's registered node. This is because post adj processing the next
 444      * node are any output features, then the midchain-tx.  from there we
 445      * need to get to the stacked child's node.
 446      */
 447     dpo_stack_from_node(tx_node,
 448                         &adj->sub_type.midchain.next_dpo,
 449                         drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 450
 451     /*
 452      * update the rewirte with the workers paused.
 453      */
 454     adj_nbr_update_rewrite_internal(adj,
 455                                     IP_LOOKUP_NEXT_MIDCHAIN,
 456                                     adj_get_midchain_node(adj->ia_link),
 457                                     tx_node,
 458                                     rewrite);
 459 }
 460
 461 /**
 462  * adj_nbr_midchain_unstack
 463  *
 464  * Unstack the adj. stack it on drop
 465  */
 466 void
 467 adj_nbr_midchain_unstack (adj_index_t adj_index)
 468 {
 469     ip_adjacency_t *adj;
 470
 471     ASSERT(ADJ_INDEX_INVALID != adj_index);
 472
 473     adj = adj_get(adj_index);
 474
 475     /*
 476      * stack on the drop
 477      */
 478     dpo_stack(DPO_ADJACENCY_MIDCHAIN,
 479               vnet_link_to_dpo_proto(adj->ia_link),
 480               &adj->sub_type.midchain.next_dpo,
 481               drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 482
 483     CLIB_MEMORY_BARRIER();
 484 }
 485
 486 /**
 487  * adj_nbr_midchain_stack
 488  */
 489 void
 490 adj_nbr_midchain_stack (adj_index_t adj_index,
 491                         const dpo_id_t *next)
 492 {
 493     ip_adjacency_t *adj;
 494
 495     ASSERT(ADJ_INDEX_INVALID != adj_index);
 496
 497     adj = adj_get(adj_index);
 498
 499     ASSERT(IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index);
 500
 501     dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj),
 502                         &adj->sub_type.midchain.next_dpo,
 503                         next);
 504 }
 505
 506 u8*
 507 format_adj_midchain (u8* s, va_list *ap)
 508 {
 509     index_t index = va_arg(*ap, index_t);
 510     u32 indent = va_arg(*ap, u32);
 511     ip_adjacency_t * adj = adj_get(index);
 512
 513     s = format (s, "%U", format_vnet_link, adj->ia_link);
 514     s = format (s, " via %U ",
 515                 format_ip46_address, &adj->sub_type.nbr.next_hop);
 516     s = format (s, " %U",
 517                 format_vnet_rewrite,
 518                 &adj->rewrite_header, sizeof (adj->rewrite_data), indent);
 519     s = format (s, "\n%Ustacked-on:\n%U%U",
 520                 format_white_space, indent,
 521                 format_white_space, indent+2,
 522                 format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
 523
 524     return (s);
 525 }
 526
 527 static void
 528 adj_dpo_lock (dpo_id_t *dpo)
 529 {
 530     adj_lock(dpo->dpoi_index);
 531 }
 532 static void
 533 adj_dpo_unlock (dpo_id_t *dpo)
 534 {
 535     adj_unlock(dpo->dpoi_index);
 536 }
 537
 538 const static dpo_vft_t adj_midchain_dpo_vft = {
 539     .dv_lock = adj_dpo_lock,
 540     .dv_unlock = adj_dpo_unlock,
 541     .dv_format = format_adj_midchain,
 542 };
 543
 544 /**
 545  * @brief The per-protocol VLIB graph nodes that are assigned to a midchain
 546  *        object.
 547  *
 548  * this means that these graph nodes are ones from which a midchain is the
 549  * parent object in the DPO-graph.
 550  */
 551 const static char* const midchain_ip4_nodes[] =
 552 {
 553     "ip4-midchain",
 554     NULL,
 555 };
 556 const static char* const midchain_ip6_nodes[] =
 557 {
 558     "ip6-midchain",
 559     NULL,
 560 };
 561 const static char* const midchain_mpls_nodes[] =
 562 {
 563     "mpls-midchain",
 564     NULL,
 565 };
 566 const static char* const midchain_ethernet_nodes[] =
 567 {
 568     "adj-l2-midchain",
 569     NULL,
 570 };
 571 const static char* const midchain_nsh_nodes[] =
 572 {
 573     "adj-nsh-midchain",
 574     NULL,
 575 };
 576
 577 const static char* const * const midchain_nodes[DPO_PROTO_NUM] =
 578 {
 579     [DPO_PROTO_IP4]  = midchain_ip4_nodes,
 580     [DPO_PROTO_IP6]  = midchain_ip6_nodes,
 581     [DPO_PROTO_MPLS] = midchain_mpls_nodes,
 582     [DPO_PROTO_ETHERNET] = midchain_ethernet_nodes,
 583     [DPO_PROTO_NSH] = midchain_nsh_nodes,
 584 };
 585
 586 void
 587 adj_midchain_module_init (void)
 588 {
 589     dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes);
 590 }