src/vnet/adj/adj_midchain.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <vnet/adj/adj_nbr.h>
  17 #include <vnet/adj/adj_internal.h>
  18 #include <vnet/adj/adj_l2.h>
  19 #include <vnet/adj/adj_midchain.h>
  20 #include <vnet/ethernet/arp_packet.h>
  21 #include <vnet/dpo/drop_dpo.h>
  22 #include <vnet/fib/fib_walk.h>
  23
  24 /**
  25  * The two midchain tx feature node indices
  26  */
  27 static u32 adj_midchain_tx_feature_node[VNET_LINK_NUM];
  28 static u32 adj_midchain_tx_no_count_feature_node[VNET_LINK_NUM];
  29
  30 /**
  31  * @brief Trace data for packets traversing the midchain tx node
  32  */
  33 typedef struct adj_midchain_tx_trace_t_
  34 {
  35     /**
  36      * @brief the midchain adj we are traversing
  37      */
  38     adj_index_t ai;
  39 } adj_midchain_tx_trace_t;
  40
  41 always_inline uword
  42 adj_midchain_tx_inline (vlib_main_t * vm,
  43                         vlib_node_runtime_t * node,
  44                         vlib_frame_t * frame,
  45                         int interface_count)
  46 {
  47     u32 * from, * to_next, n_left_from, n_left_to_next;
  48     u32 next_index;
  49     vnet_main_t *vnm = vnet_get_main ();
  50     vnet_interface_main_t *im = &vnm->interface_main;
  51     u32 cpu_index = vm->cpu_index;
  52
  53     /* Vector of buffer / pkt indices we're supposed to process */
  54     from = vlib_frame_vector_args (frame);
  55
  56     /* Number of buffers / pkts */
  57     n_left_from = frame->n_vectors;
  58
  59     /* Speculatively send the first buffer to the last disposition we used */
  60     next_index = node->cached_next_index;
  61
  62     while (n_left_from > 0)
  63     {
  64         /* set up to enqueue to our disposition with index = next_index */
  65         vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
  66
  67
  68         while (n_left_from >= 4 && n_left_to_next > 2)
  69         {
  70             u32 bi0, adj_index0, next0;
  71             const ip_adjacency_t * adj0;
  72             const dpo_id_t *dpo0;
  73             vlib_buffer_t * b0;
  74             u32 bi1, adj_index1, next1;
  75             const ip_adjacency_t * adj1;
  76             const dpo_id_t *dpo1;
  77             vlib_buffer_t * b1;
  78
  79             /* Prefetch next iteration. */
  80             {
  81                 vlib_buffer_t * p2, * p3;
  82
  83                 p2 = vlib_get_buffer (vm, from[2]);
  84                 p3 = vlib_get_buffer (vm, from[3]);
  85
  86                 vlib_prefetch_buffer_header (p2, LOAD);
  87                 vlib_prefetch_buffer_header (p3, LOAD);
  88
  89                 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
  90                 CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
  91             }
  92
  93             bi0 = from[0];
  94             to_next[0] = bi0;
  95             bi1 = from[1];
  96             to_next[1] = bi1;
  97
  98             from += 2;
  99             to_next += 2;
 100             n_left_from -= 2;
 101             n_left_to_next -= 2;
 102
 103             b0 = vlib_get_buffer(vm, bi0);
 104             b1 = vlib_get_buffer(vm, bi1);
 105
 106             /* Follow the DPO on which the midchain is stacked */
 107             adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 108             adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
 109
 110             adj0 = adj_get(adj_index0);
 111             adj1 = adj_get(adj_index1);
 112
 113             dpo0 = &adj0->sub_type.midchain.next_dpo;
 114             dpo1 = &adj1->sub_type.midchain.next_dpo;
 115
 116             next0 = dpo0->dpoi_next_node;
 117             next1 = dpo1->dpoi_next_node;
 118
 119             vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 120             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 121
 122             if (interface_count)
 123             {
 124                 vlib_increment_combined_counter (im->combined_sw_if_counters
 125                                                  + VNET_INTERFACE_COUNTER_TX,
 126                                                  cpu_index,
 127                                                  adj0->rewrite_header.sw_if_index,
 128                                                  1,
 129                                                  vlib_buffer_length_in_chain (vm, b0));
 130                 vlib_increment_combined_counter (im->combined_sw_if_counters
 131                                                  + VNET_INTERFACE_COUNTER_TX,
 132                                                  cpu_index,
 133                                                  adj1->rewrite_header.sw_if_index,
 134                                                  1,
 135                                                  vlib_buffer_length_in_chain (vm, b1));
 136             }
 137
 138             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 139             {
 140                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 141                                                               b0, sizeof (*tr));
 142                 tr->ai = adj_index0;
 143             }
 144             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 145             {
 146                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 147                                                               b1, sizeof (*tr));
 148                 tr->ai = adj_index1;
 149             }
 150
 151             vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 152                                              to_next, n_left_to_next,
 153                                              bi0, bi1,
 154                                              next0, next1);
 155         }
 156         while (n_left_from > 0 && n_left_to_next > 0)
 157         {
 158             u32 bi0, adj_index0, next0;
 159             const ip_adjacency_t * adj0;
 160             const dpo_id_t *dpo0;
 161             vlib_buffer_t * b0;
 162
 163             bi0 = from[0];
 164             to_next[0] = bi0;
 165             from += 1;
 166             to_next += 1;
 167             n_left_from -= 1;
 168             n_left_to_next -= 1;
 169
 170             b0 = vlib_get_buffer(vm, bi0);
 171
 172             /* Follow the DPO on which the midchain is stacked */
 173             adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
 174             adj0 = adj_get(adj_index0);
 175             dpo0 = &adj0->sub_type.midchain.next_dpo;
 176             next0 = dpo0->dpoi_next_node;
 177             vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 178
 179             if (interface_count)
 180             {
 181                 vlib_increment_combined_counter (im->combined_sw_if_counters
 182                                                  + VNET_INTERFACE_COUNTER_TX,
 183                                                  cpu_index,
 184                                                  adj0->rewrite_header.sw_if_index,
 185                                                  1,
 186                                                  vlib_buffer_length_in_chain (vm, b0));
 187             }
 188
 189             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 190             {
 191                 adj_midchain_tx_trace_t *tr = vlib_add_trace (vm, node,
 192                                                               b0, sizeof (*tr));
 193                 tr->ai = adj_index0;
 194             }
 195
 196             vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 197                                              to_next, n_left_to_next,
 198                                              bi0, next0);
 199         }
 200
 201         vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 202     }
 203
 204     return frame->n_vectors;
 205 }
 206
 207 static u8 *
 208 format_adj_midchain_tx_trace (u8 * s, va_list * args)
 209 {
 210     CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 211     CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 212     adj_midchain_tx_trace_t *tr = va_arg (*args, adj_midchain_tx_trace_t*);
 213
 214     s = format(s, "adj-midchain:[%d]:%U", tr->ai,
 215                format_ip_adjacency, tr->ai,
 216                FORMAT_IP_ADJACENCY_NONE);
 217
 218     return (s);
 219 }
 220
 221 static uword
 222 adj_midchain_tx (vlib_main_t * vm,
 223                  vlib_node_runtime_t * node,
 224                  vlib_frame_t * frame)
 225 {
 226     return (adj_midchain_tx_inline(vm, node, frame, 1));
 227 }
 228
 229 VLIB_REGISTER_NODE (adj_midchain_tx_node, static) = {
 230     .function = adj_midchain_tx,
 231     .name = "adj-midchain-tx",
 232     .vector_size = sizeof (u32),
 233
 234     .format_trace = format_adj_midchain_tx_trace,
 235
 236     .n_next_nodes = 1,
 237     .next_nodes = {
 238         [0] = "error-drop",
 239     },
 240 };
 241
 242 static uword
 243 adj_midchain_tx_no_count (vlib_main_t * vm,
 244                           vlib_node_runtime_t * node,
 245                           vlib_frame_t * frame)
 246 {
 247     return (adj_midchain_tx_inline(vm, node, frame, 0));
 248 }
 249
 250 VLIB_REGISTER_NODE (adj_midchain_tx_no_count_node, static) = {
 251     .function = adj_midchain_tx_no_count,
 252     .name = "adj-midchain-tx-no-count",
 253     .vector_size = sizeof (u32),
 254
 255     .format_trace = format_adj_midchain_tx_trace,
 256
 257     .n_next_nodes = 1,
 258     .next_nodes = {
 259         [0] = "error-drop",
 260     },
 261 };
 262
 263 VNET_FEATURE_INIT (adj_midchain_tx_ip4, static) = {
 264     .arc_name = "ip4-output",
 265     .node_name = "adj-midchain-tx",
 266     .runs_before = VNET_FEATURES ("interface-output"),
 267     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP4],
 268 };
 269 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip4, static) = {
 270     .arc_name = "ip4-output",
 271     .node_name = "adj-midchain-tx-no-count",
 272     .runs_before = VNET_FEATURES ("interface-output"),
 273     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP4],
 274 };
 275 VNET_FEATURE_INIT (adj_midchain_tx_ip6, static) = {
 276     .arc_name = "ip6-output",
 277     .node_name = "adj-midchain-tx",
 278     .runs_before = VNET_FEATURES ("interface-output"),
 279     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_IP6],
 280 };
 281 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ip6, static) = {
 282     .arc_name = "ip6-output",
 283     .node_name = "adj-midchain-tx-no-count",
 284     .runs_before = VNET_FEATURES ("interface-output"),
 285     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_IP6],
 286 };
 287 VNET_FEATURE_INIT (adj_midchain_tx_mpls, static) = {
 288     .arc_name = "mpls-output",
 289     .node_name = "adj-midchain-tx",
 290     .runs_before = VNET_FEATURES ("interface-output"),
 291     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_MPLS],
 292 };
 293 VNET_FEATURE_INIT (adj_midchain_tx_no_count_mpls, static) = {
 294     .arc_name = "mpls-output",
 295     .node_name = "adj-midchain-tx-no-count",
 296     .runs_before = VNET_FEATURES ("interface-output"),
 297     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_MPLS],
 298 };
 299 VNET_FEATURE_INIT (adj_midchain_tx_ethernet, static) = {
 300     .arc_name = "ethernet-output",
 301     .node_name = "adj-midchain-tx",
 302     .runs_before = VNET_FEATURES ("error-drop"),
 303     .feature_index_ptr = &adj_midchain_tx_feature_node[VNET_LINK_ETHERNET],
 304 };
 305 VNET_FEATURE_INIT (adj_midchain_tx_no_count_ethernet, static) = {
 306     .arc_name = "ethernet-output",
 307     .node_name = "adj-midchain-tx-no-count",
 308     .runs_before = VNET_FEATURES ("error-drop"),
 309     .feature_index_ptr = &adj_midchain_tx_no_count_feature_node[VNET_LINK_ETHERNET],
 310 };
 311
 312 static inline u32
 313 adj_get_midchain_node (vnet_link_t link)
 314 {
 315     switch (link) {
 316     case VNET_LINK_IP4:
 317         return (ip4_midchain_node.index);
 318     case VNET_LINK_IP6:
 319         return (ip6_midchain_node.index);
 320     case VNET_LINK_MPLS:
 321         return (mpls_midchain_node.index);
 322     case VNET_LINK_ETHERNET:
 323         return (adj_l2_midchain_node.index);
 324     case VNET_LINK_ARP:
 325         break;
 326     }
 327     ASSERT(0);
 328     return (0);
 329 }
 330
 331 static u8
 332 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
 333 {
 334   u8 arc = (u8) ~0;
 335     switch (adj->ia_link)
 336     {
 337     case VNET_LINK_IP4:
 338         {
 339             arc = ip4_main.lookup_main.output_feature_arc_index;
 340             break;
 341         }
 342     case VNET_LINK_IP6:
 343         {
 344             arc = ip6_main.lookup_main.output_feature_arc_index;
 345             break;
 346         }
 347     case VNET_LINK_MPLS:
 348         {
 349             arc = mpls_main.output_feature_arc_index;
 350             break;
 351         }
 352     case VNET_LINK_ETHERNET:
 353         {
 354             arc = ethernet_main.output_feature_arc_index;
 355             break;
 356         }
 357     case VNET_LINK_ARP:
 358         ASSERT(0);
 359         break;
 360     }
 361
 362     ASSERT (arc != (u8) ~0);
 363
 364     return (arc);
 365 }
 366
 367 /**
 368  * adj_nbr_midchain_update_rewrite
 369  *
 370  * Update the adjacency's rewrite string. A NULL string implies the
 371  * rewrite is reset (i.e. when ARP/ND etnry is gone).
 372  * NB: the adj being updated may be handling traffic in the DP.
 373  */
 374 void
 375 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
 376                                  adj_midchain_fixup_t fixup,
 377                                  adj_midchain_flag_t flags,
 378                                  u8 *rewrite)
 379 {
 380     ip_adjacency_t *adj;
 381     u8 arc_index;
 382     u32 feature_index;
 383
 384     ASSERT(ADJ_INDEX_INVALID != adj_index);
 385
 386     adj = adj_get(adj_index);
 387
 388     /*
 389      * one time only update. since we don't support chainging the tunnel
 390      * src,dst, this is all we need.
 391      */
 392     ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP);
 393     /*
 394      * tunnels can always provide a rewrite.
 395      */
 396     ASSERT(NULL != rewrite);
 397
 398     adj->sub_type.midchain.fixup_func = fixup;
 399
 400     arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
 401     feature_index = (flags & ADJ_MIDCHAIN_FLAG_NO_COUNT) ?
 402                     adj_midchain_tx_no_count_feature_node[adj->ia_link] :
 403                     adj_midchain_tx_feature_node[adj->ia_link];
 404
 405     adj->sub_type.midchain.tx_function_node = (flags & ADJ_MIDCHAIN_FLAG_NO_COUNT) ?
 406                                                adj_midchain_tx_no_count_node.index :
 407                                                adj_midchain_tx_node.index;
 408
 409     vnet_feature_enable_disable_with_index (arc_index, feature_index,
 410                                             adj->rewrite_header.sw_if_index,
 411                                             1 /* enable */, 0, 0);
 412
 413     /*
 414      * stack the midchain on the drop so it's ready to forward in the adj-midchain-tx.
 415      * The graph arc used/created here is from the midchain-tx node to the
 416      * child's registered node. This is because post adj processing the next
 417      * node are any output features, then the midchain-tx.  from there we
 418      * need to get to the stacked child's node.
 419      */
 420     dpo_stack_from_node(adj->sub_type.midchain.tx_function_node,
 421                         &adj->sub_type.midchain.next_dpo,
 422                         drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 423
 424     /*
 425      * update the rewirte with the workers paused.
 426      */
 427     adj_nbr_update_rewrite_internal(adj,
 428                                     IP_LOOKUP_NEXT_MIDCHAIN,
 429                                     adj_get_midchain_node(adj->ia_link),
 430                                     adj->sub_type.midchain.tx_function_node,
 431                                     rewrite);
 432 }
 433
 434 /**
 435  * adj_nbr_midchain_unstack
 436  *
 437  * Unstack the adj. stack it on drop
 438  */
 439 void
 440 adj_nbr_midchain_unstack (adj_index_t adj_index)
 441 {
 442     ip_adjacency_t *adj;
 443
 444     ASSERT(ADJ_INDEX_INVALID != adj_index);
 445
 446     adj = adj_get(adj_index);
 447
 448     /*
 449      * stack on the drop
 450      */
 451     dpo_stack(DPO_ADJACENCY_MIDCHAIN,
 452               vnet_link_to_dpo_proto(adj->ia_link),
 453               &adj->sub_type.midchain.next_dpo,
 454               drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 455
 456     CLIB_MEMORY_BARRIER();
 457 }
 458
 459 /**
 460  * adj_nbr_midchain_stack
 461  */
 462 void
 463 adj_nbr_midchain_stack (adj_index_t adj_index,
 464                         const dpo_id_t *next)
 465 {
 466     ip_adjacency_t *adj;
 467
 468     ASSERT(ADJ_INDEX_INVALID != adj_index);
 469
 470     adj = adj_get(adj_index);
 471
 472     ASSERT(IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index);
 473
 474     dpo_stack_from_node(adj->sub_type.midchain.tx_function_node,
 475                         &adj->sub_type.midchain.next_dpo,
 476                         next);
 477 }
 478
 479 u8*
 480 format_adj_midchain (u8* s, va_list *ap)
 481 {
 482     index_t index = va_arg(*ap, index_t);
 483     u32 indent = va_arg(*ap, u32);
 484     vnet_main_t * vnm = vnet_get_main();
 485     ip_adjacency_t * adj = adj_get(index);
 486
 487     s = format (s, "%U", format_vnet_link, adj->ia_link);
 488     s = format (s, " via %U ",
 489                 format_ip46_address, &adj->sub_type.nbr.next_hop);
 490     s = format (s, " %U",
 491                 format_vnet_rewrite,
 492                 vnm->vlib_main, &adj->rewrite_header,
 493                 sizeof (adj->rewrite_data), indent);
 494     s = format (s, "\n%Ustacked-on:\n%U%U",
 495                 format_white_space, indent,
 496                 format_white_space, indent+2,
 497                 format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
 498
 499     return (s);
 500 }
 501
 502 static void
 503 adj_dpo_lock (dpo_id_t *dpo)
 504 {
 505     adj_lock(dpo->dpoi_index);
 506 }
 507 static void
 508 adj_dpo_unlock (dpo_id_t *dpo)
 509 {
 510     adj_unlock(dpo->dpoi_index);
 511 }
 512
 513 const static dpo_vft_t adj_midchain_dpo_vft = {
 514     .dv_lock = adj_dpo_lock,
 515     .dv_unlock = adj_dpo_unlock,
 516     .dv_format = format_adj_midchain,
 517 };
 518
 519 /**
 520  * @brief The per-protocol VLIB graph nodes that are assigned to a midchain
 521  *        object.
 522  *
 523  * this means that these graph nodes are ones from which a midchain is the
 524  * parent object in the DPO-graph.
 525  */
 526 const static char* const midchain_ip4_nodes[] =
 527 {
 528     "ip4-midchain",
 529     NULL,
 530 };
 531 const static char* const midchain_ip6_nodes[] =
 532 {
 533     "ip6-midchain",
 534     NULL,
 535 };
 536 const static char* const midchain_mpls_nodes[] =
 537 {
 538     "mpls-midchain",
 539     NULL,
 540 };
 541 const static char* const midchain_ethernet_nodes[] =
 542 {
 543     "adj-l2-midchain",
 544     NULL,
 545 };
 546
 547 const static char* const * const midchain_nodes[DPO_PROTO_NUM] =
 548 {
 549     [DPO_PROTO_IP4]  = midchain_ip4_nodes,
 550     [DPO_PROTO_IP6]  = midchain_ip6_nodes,
 551     [DPO_PROTO_MPLS] = midchain_mpls_nodes,
 552     [DPO_PROTO_ETHERNET] = midchain_ethernet_nodes,
 553 };
 554
 555 void
 556 adj_midchain_module_init (void)
 557 {
 558     dpo_register(DPO_ADJACENCY_MIDCHAIN, &adj_midchain_dpo_vft, midchain_nodes);
 559 }