src/vnet/ip/ip4_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip4_forward.c: IP v4 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  43 #include <vnet/ethernet/arp_packet.h>   /* for ethernet_arp_header_t */
  44 #include <vnet/ppp/ppp.h>
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vnet/api_errno.h>     /* for API error numbers */
  47 #include <vnet/fib/fib_table.h> /* for FIB table and entry creation */
  48 #include <vnet/fib/fib_entry.h> /* for FIB table and entry creation */
  49 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  50 #include <vnet/fib/ip4_fib.h>
  51 #include <vnet/dpo/load_balance.h>
  52 #include <vnet/dpo/load_balance_map.h>
  53 #include <vnet/dpo/classify_dpo.h>
  54 #include <vnet/mfib/mfib_table.h>       /* for mFIB table and entry creation */
  55
  56 /**
  57  * @file
  58  * @brief IPv4 Forwarding.
  59  *
  60  * This file contains the source code for IPv4 forwarding.
  61  */
  62
  63 void
  64 ip4_forward_next_trace (vlib_main_t * vm,
  65                         vlib_node_runtime_t * node,
  66                         vlib_frame_t * frame,
  67                         vlib_rx_or_tx_t which_adj_index);
  68
  69 always_inline uword
  70 ip4_lookup_inline (vlib_main_t * vm,
  71                    vlib_node_runtime_t * node,
  72                    vlib_frame_t * frame,
  73                    int lookup_for_responses_to_locally_received_packets)
  74 {
  75   ip4_main_t *im = &ip4_main;
  76   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters;
  77   u32 n_left_from, n_left_to_next, *from, *to_next;
  78   ip_lookup_next_t next;
  79   u32 thread_index = vlib_get_thread_index ();
  80
  81   from = vlib_frame_vector_args (frame);
  82   n_left_from = frame->n_vectors;
  83   next = node->cached_next_index;
  84
  85   while (n_left_from > 0)
  86     {
  87       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
  88
  89       while (n_left_from >= 8 && n_left_to_next >= 4)
  90         {
  91           vlib_buffer_t *p0, *p1, *p2, *p3;
  92           ip4_header_t *ip0, *ip1, *ip2, *ip3;
  93           ip_lookup_next_t next0, next1, next2, next3;
  94           const load_balance_t *lb0, *lb1, *lb2, *lb3;
  95           ip4_fib_mtrie_t *mtrie0, *mtrie1, *mtrie2, *mtrie3;
  96           ip4_fib_mtrie_leaf_t leaf0, leaf1, leaf2, leaf3;
  97           ip4_address_t *dst_addr0, *dst_addr1, *dst_addr2, *dst_addr3;
  98           u32 pi0, fib_index0, lb_index0;
  99           u32 pi1, fib_index1, lb_index1;
 100           u32 pi2, fib_index2, lb_index2;
 101           u32 pi3, fib_index3, lb_index3;
 102           flow_hash_config_t flow_hash_config0, flow_hash_config1;
 103           flow_hash_config_t flow_hash_config2, flow_hash_config3;
 104           u32 hash_c0, hash_c1, hash_c2, hash_c3;
 105           const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3;
 106
 107           /* Prefetch next iteration. */
 108           {
 109             vlib_buffer_t *p4, *p5, *p6, *p7;
 110
 111             p4 = vlib_get_buffer (vm, from[4]);
 112             p5 = vlib_get_buffer (vm, from[5]);
 113             p6 = vlib_get_buffer (vm, from[6]);
 114             p7 = vlib_get_buffer (vm, from[7]);
 115
 116             vlib_prefetch_buffer_header (p4, LOAD);
 117             vlib_prefetch_buffer_header (p5, LOAD);
 118             vlib_prefetch_buffer_header (p6, LOAD);
 119             vlib_prefetch_buffer_header (p7, LOAD);
 120
 121             CLIB_PREFETCH (p4->data, sizeof (ip0[0]), LOAD);
 122             CLIB_PREFETCH (p5->data, sizeof (ip0[0]), LOAD);
 123             CLIB_PREFETCH (p6->data, sizeof (ip0[0]), LOAD);
 124             CLIB_PREFETCH (p7->data, sizeof (ip0[0]), LOAD);
 125           }
 126
 127           pi0 = to_next[0] = from[0];
 128           pi1 = to_next[1] = from[1];
 129           pi2 = to_next[2] = from[2];
 130           pi3 = to_next[3] = from[3];
 131
 132           from += 4;
 133           to_next += 4;
 134           n_left_to_next -= 4;
 135           n_left_from -= 4;
 136
 137           p0 = vlib_get_buffer (vm, pi0);
 138           p1 = vlib_get_buffer (vm, pi1);
 139           p2 = vlib_get_buffer (vm, pi2);
 140           p3 = vlib_get_buffer (vm, pi3);
 141
 142           ip0 = vlib_buffer_get_current (p0);
 143           ip1 = vlib_buffer_get_current (p1);
 144           ip2 = vlib_buffer_get_current (p2);
 145           ip3 = vlib_buffer_get_current (p3);
 146
 147           dst_addr0 = &ip0->dst_address;
 148           dst_addr1 = &ip1->dst_address;
 149           dst_addr2 = &ip2->dst_address;
 150           dst_addr3 = &ip3->dst_address;
 151
 152           fib_index0 =
 153             vec_elt (im->fib_index_by_sw_if_index,
 154                      vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 155           fib_index1 =
 156             vec_elt (im->fib_index_by_sw_if_index,
 157                      vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 158           fib_index2 =
 159             vec_elt (im->fib_index_by_sw_if_index,
 160                      vnet_buffer (p2)->sw_if_index[VLIB_RX]);
 161           fib_index3 =
 162             vec_elt (im->fib_index_by_sw_if_index,
 163                      vnet_buffer (p3)->sw_if_index[VLIB_RX]);
 164           fib_index0 =
 165             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
 166              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 167           fib_index1 =
 168             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
 169              (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
 170           fib_index2 =
 171             (vnet_buffer (p2)->sw_if_index[VLIB_TX] ==
 172              (u32) ~ 0) ? fib_index2 : vnet_buffer (p2)->sw_if_index[VLIB_TX];
 173           fib_index3 =
 174             (vnet_buffer (p3)->sw_if_index[VLIB_TX] ==
 175              (u32) ~ 0) ? fib_index3 : vnet_buffer (p3)->sw_if_index[VLIB_TX];
 176
 177
 178           if (!lookup_for_responses_to_locally_received_packets)
 179             {
 180               mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 181               mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
 182               mtrie2 = &ip4_fib_get (fib_index2)->mtrie;
 183               mtrie3 = &ip4_fib_get (fib_index3)->mtrie;
 184
 185               leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
 186               leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, dst_addr1);
 187               leaf2 = ip4_fib_mtrie_lookup_step_one (mtrie2, dst_addr2);
 188               leaf3 = ip4_fib_mtrie_lookup_step_one (mtrie3, dst_addr3);
 189             }
 190
 191           if (!lookup_for_responses_to_locally_received_packets)
 192             {
 193               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 194               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
 195               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 2);
 196               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 2);
 197             }
 198
 199           if (!lookup_for_responses_to_locally_received_packets)
 200             {
 201               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 202               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
 203               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 3);
 204               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 3);
 205             }
 206
 207           if (lookup_for_responses_to_locally_received_packets)
 208             {
 209               lb_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 210               lb_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
 211               lb_index2 = vnet_buffer (p2)->ip.adj_index[VLIB_RX];
 212               lb_index3 = vnet_buffer (p3)->ip.adj_index[VLIB_RX];
 213             }
 214           else
 215             {
 216               lb_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 217               lb_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 218               lb_index2 = ip4_fib_mtrie_leaf_get_adj_index (leaf2);
 219               lb_index3 = ip4_fib_mtrie_leaf_get_adj_index (leaf3);
 220             }
 221
 222           ASSERT (lb_index0 && lb_index1 && lb_index2 && lb_index3);
 223           lb0 = load_balance_get (lb_index0);
 224           lb1 = load_balance_get (lb_index1);
 225           lb2 = load_balance_get (lb_index2);
 226           lb3 = load_balance_get (lb_index3);
 227
 228           ASSERT (lb0->lb_n_buckets > 0);
 229           ASSERT (is_pow2 (lb0->lb_n_buckets));
 230           ASSERT (lb1->lb_n_buckets > 0);
 231           ASSERT (is_pow2 (lb1->lb_n_buckets));
 232           ASSERT (lb2->lb_n_buckets > 0);
 233           ASSERT (is_pow2 (lb2->lb_n_buckets));
 234           ASSERT (lb3->lb_n_buckets > 0);
 235           ASSERT (is_pow2 (lb3->lb_n_buckets));
 236
 237           /* Use flow hash to compute multipath adjacency. */
 238           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 239           hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
 240           hash_c2 = vnet_buffer (p2)->ip.flow_hash = 0;
 241           hash_c3 = vnet_buffer (p3)->ip.flow_hash = 0;
 242           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 243             {
 244               flow_hash_config0 = lb0->lb_hash_config;
 245               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 246                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 247               dpo0 =
 248                 load_balance_get_fwd_bucket (lb0,
 249                                              (hash_c0 &
 250                                               (lb0->lb_n_buckets_minus_1)));
 251             }
 252           else
 253             {
 254               dpo0 = load_balance_get_bucket_i (lb0, 0);
 255             }
 256           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 257             {
 258               flow_hash_config1 = lb1->lb_hash_config;
 259               hash_c1 = vnet_buffer (p1)->ip.flow_hash =
 260                 ip4_compute_flow_hash (ip1, flow_hash_config1);
 261               dpo1 =
 262                 load_balance_get_fwd_bucket (lb1,
 263                                              (hash_c1 &
 264                                               (lb1->lb_n_buckets_minus_1)));
 265             }
 266           else
 267             {
 268               dpo1 = load_balance_get_bucket_i (lb1, 0);
 269             }
 270           if (PREDICT_FALSE (lb2->lb_n_buckets > 1))
 271             {
 272               flow_hash_config2 = lb2->lb_hash_config;
 273               hash_c2 = vnet_buffer (p2)->ip.flow_hash =
 274                 ip4_compute_flow_hash (ip2, flow_hash_config2);
 275               dpo2 =
 276                 load_balance_get_fwd_bucket (lb2,
 277                                              (hash_c2 &
 278                                               (lb2->lb_n_buckets_minus_1)));
 279             }
 280           else
 281             {
 282               dpo2 = load_balance_get_bucket_i (lb2, 0);
 283             }
 284           if (PREDICT_FALSE (lb3->lb_n_buckets > 1))
 285             {
 286               flow_hash_config3 = lb3->lb_hash_config;
 287               hash_c3 = vnet_buffer (p3)->ip.flow_hash =
 288                 ip4_compute_flow_hash (ip3, flow_hash_config3);
 289               dpo3 =
 290                 load_balance_get_fwd_bucket (lb3,
 291                                              (hash_c3 &
 292                                               (lb3->lb_n_buckets_minus_1)));
 293             }
 294           else
 295             {
 296               dpo3 = load_balance_get_bucket_i (lb3, 0);
 297             }
 298
 299           next0 = dpo0->dpoi_next_node;
 300           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 301           next1 = dpo1->dpoi_next_node;
 302           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 303           next2 = dpo2->dpoi_next_node;
 304           vnet_buffer (p2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
 305           next3 = dpo3->dpoi_next_node;
 306           vnet_buffer (p3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
 307
 308           vlib_increment_combined_counter
 309             (cm, thread_index, lb_index0, 1,
 310              vlib_buffer_length_in_chain (vm, p0));
 311           vlib_increment_combined_counter
 312             (cm, thread_index, lb_index1, 1,
 313              vlib_buffer_length_in_chain (vm, p1));
 314           vlib_increment_combined_counter
 315             (cm, thread_index, lb_index2, 1,
 316              vlib_buffer_length_in_chain (vm, p2));
 317           vlib_increment_combined_counter
 318             (cm, thread_index, lb_index3, 1,
 319              vlib_buffer_length_in_chain (vm, p3));
 320
 321           vlib_validate_buffer_enqueue_x4 (vm, node, next,
 322                                            to_next, n_left_to_next,
 323                                            pi0, pi1, pi2, pi3,
 324                                            next0, next1, next2, next3);
 325         }
 326
 327       while (n_left_from > 0 && n_left_to_next > 0)
 328         {
 329           vlib_buffer_t *p0;
 330           ip4_header_t *ip0;
 331           ip_lookup_next_t next0;
 332           const load_balance_t *lb0;
 333           ip4_fib_mtrie_t *mtrie0;
 334           ip4_fib_mtrie_leaf_t leaf0;
 335           ip4_address_t *dst_addr0;
 336           u32 pi0, fib_index0, lbi0;
 337           flow_hash_config_t flow_hash_config0;
 338           const dpo_id_t *dpo0;
 339           u32 hash_c0;
 340
 341           pi0 = from[0];
 342           to_next[0] = pi0;
 343
 344           p0 = vlib_get_buffer (vm, pi0);
 345
 346           ip0 = vlib_buffer_get_current (p0);
 347
 348           dst_addr0 = &ip0->dst_address;
 349
 350           fib_index0 =
 351             vec_elt (im->fib_index_by_sw_if_index,
 352                      vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 353           fib_index0 =
 354             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
 355              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 356
 357           if (!lookup_for_responses_to_locally_received_packets)
 358             {
 359               mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 360
 361               leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
 362             }
 363
 364           if (!lookup_for_responses_to_locally_received_packets)
 365             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 366
 367           if (!lookup_for_responses_to_locally_received_packets)
 368             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 369
 370           if (lookup_for_responses_to_locally_received_packets)
 371             lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 372           else
 373             {
 374               /* Handle default route. */
 375               lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 376             }
 377
 378           ASSERT (lbi0);
 379           lb0 = load_balance_get (lbi0);
 380
 381           ASSERT (lb0->lb_n_buckets > 0);
 382           ASSERT (is_pow2 (lb0->lb_n_buckets));
 383
 384           /* Use flow hash to compute multipath adjacency. */
 385           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 386           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 387             {
 388               flow_hash_config0 = lb0->lb_hash_config;
 389
 390               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 391                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 392               dpo0 =
 393                 load_balance_get_fwd_bucket (lb0,
 394                                              (hash_c0 &
 395                                               (lb0->lb_n_buckets_minus_1)));
 396             }
 397           else
 398             {
 399               dpo0 = load_balance_get_bucket_i (lb0, 0);
 400             }
 401
 402           next0 = dpo0->dpoi_next_node;
 403           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 404
 405           vlib_increment_combined_counter (cm, thread_index, lbi0, 1,
 406                                            vlib_buffer_length_in_chain (vm,
 407                                                                         p0));
 408
 409           from += 1;
 410           to_next += 1;
 411           n_left_to_next -= 1;
 412           n_left_from -= 1;
 413
 414           if (PREDICT_FALSE (next0 != next))
 415             {
 416               n_left_to_next += 1;
 417               vlib_put_next_frame (vm, node, next, n_left_to_next);
 418               next = next0;
 419               vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 420               to_next[0] = pi0;
 421               to_next += 1;
 422               n_left_to_next -= 1;
 423             }
 424         }
 425
 426       vlib_put_next_frame (vm, node, next, n_left_to_next);
 427     }
 428
 429   if (node->flags & VLIB_NODE_FLAG_TRACE)
 430     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 431
 432   return frame->n_vectors;
 433 }
 434
 435 /** @brief IPv4 lookup node.
 436     @node ip4-lookup
 437
 438     This is the main IPv4 lookup dispatch node.
 439
 440     @param vm vlib_main_t corresponding to the current thread
 441     @param node vlib_node_runtime_t
 442     @param frame vlib_frame_t whose contents should be dispatched
 443
 444     @par Graph mechanics: buffer metadata, next index usage
 445
 446     @em Uses:
 447     - <code>vnet_buffer(b)->sw_if_index[VLIB_RX]</code>
 448         - Indicates the @c sw_if_index value of the interface that the
 449           packet was received on.
 450     - <code>vnet_buffer(b)->sw_if_index[VLIB_TX]</code>
 451         - When the value is @c ~0 then the node performs a longest prefix
 452           match (LPM) for the packet destination address in the FIB attached
 453           to the receive interface.
 454         - Otherwise perform LPM for the packet destination address in the
 455           indicated FIB. In this case <code>[VLIB_TX]</code> is a FIB index
 456           value (0, 1, ...) and not a VRF id.
 457
 458     @em Sets:
 459     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
 460         - The lookup result adjacency index.
 461
 462     <em>Next Index:</em>
 463     - Dispatches the packet to the node index found in
 464       ip_adjacency_t @c adj->lookup_next_index
 465       (where @c adj is the lookup result adjacency).
 466 */
 467 static uword
 468 ip4_lookup (vlib_main_t * vm,
 469             vlib_node_runtime_t * node, vlib_frame_t * frame)
 470 {
 471   return ip4_lookup_inline (vm, node, frame,
 472                             /* lookup_for_responses_to_locally_received_packets */
 473                             0);
 474
 475 }
 476
 477 static u8 *format_ip4_lookup_trace (u8 * s, va_list * args);
 478
 479 VLIB_REGISTER_NODE (ip4_lookup_node) =
 480 {
 481 .function = ip4_lookup,.name = "ip4-lookup",.vector_size =
 482     sizeof (u32),.format_trace = format_ip4_lookup_trace,.n_next_nodes =
 483     IP_LOOKUP_N_NEXT,.next_nodes = IP4_LOOKUP_NEXT_NODES,};
 484
 485 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup);
 486
 487 always_inline uword
 488 ip4_load_balance (vlib_main_t * vm,
 489                   vlib_node_runtime_t * node, vlib_frame_t * frame)
 490 {
 491   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 492   u32 n_left_from, n_left_to_next, *from, *to_next;
 493   ip_lookup_next_t next;
 494   u32 thread_index = vlib_get_thread_index ();
 495
 496   from = vlib_frame_vector_args (frame);
 497   n_left_from = frame->n_vectors;
 498   next = node->cached_next_index;
 499
 500   if (node->flags & VLIB_NODE_FLAG_TRACE)
 501     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 502
 503   while (n_left_from > 0)
 504     {
 505       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 506
 507
 508       while (n_left_from >= 4 && n_left_to_next >= 2)
 509         {
 510           ip_lookup_next_t next0, next1;
 511           const load_balance_t *lb0, *lb1;
 512           vlib_buffer_t *p0, *p1;
 513           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 514           const ip4_header_t *ip0, *ip1;
 515           const dpo_id_t *dpo0, *dpo1;
 516
 517           /* Prefetch next iteration. */
 518           {
 519             vlib_buffer_t *p2, *p3;
 520
 521             p2 = vlib_get_buffer (vm, from[2]);
 522             p3 = vlib_get_buffer (vm, from[3]);
 523
 524             vlib_prefetch_buffer_header (p2, STORE);
 525             vlib_prefetch_buffer_header (p3, STORE);
 526
 527             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 528             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 529           }
 530
 531           pi0 = to_next[0] = from[0];
 532           pi1 = to_next[1] = from[1];
 533
 534           from += 2;
 535           n_left_from -= 2;
 536           to_next += 2;
 537           n_left_to_next -= 2;
 538
 539           p0 = vlib_get_buffer (vm, pi0);
 540           p1 = vlib_get_buffer (vm, pi1);
 541
 542           ip0 = vlib_buffer_get_current (p0);
 543           ip1 = vlib_buffer_get_current (p1);
 544           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 545           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 546
 547           lb0 = load_balance_get (lbi0);
 548           lb1 = load_balance_get (lbi1);
 549
 550           /*
 551            * this node is for via FIBs we can re-use the hash value from the
 552            * to node if present.
 553            * We don't want to use the same hash value at each level in the recursion
 554            * graph as that would lead to polarisation
 555            */
 556           hc0 = hc1 = 0;
 557
 558           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 559             {
 560               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 561                 {
 562                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 563                     vnet_buffer (p0)->ip.flow_hash >> 1;
 564                 }
 565               else
 566                 {
 567                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 568                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 569                 }
 570               dpo0 = load_balance_get_fwd_bucket
 571                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 572             }
 573           else
 574             {
 575               dpo0 = load_balance_get_bucket_i (lb0, 0);
 576             }
 577           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 578             {
 579               if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash))
 580                 {
 581                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 582                     vnet_buffer (p1)->ip.flow_hash >> 1;
 583                 }
 584               else
 585                 {
 586                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 587                     ip4_compute_flow_hash (ip1, lb1->lb_hash_config);
 588                 }
 589               dpo1 = load_balance_get_fwd_bucket
 590                 (lb1, (hc1 & (lb1->lb_n_buckets_minus_1)));
 591             }
 592           else
 593             {
 594               dpo1 = load_balance_get_bucket_i (lb1, 0);
 595             }
 596
 597           next0 = dpo0->dpoi_next_node;
 598           next1 = dpo1->dpoi_next_node;
 599
 600           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 601           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 602
 603           vlib_increment_combined_counter
 604             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 605           vlib_increment_combined_counter
 606             (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 607
 608           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 609                                            to_next, n_left_to_next,
 610                                            pi0, pi1, next0, next1);
 611         }
 612
 613       while (n_left_from > 0 && n_left_to_next > 0)
 614         {
 615           ip_lookup_next_t next0;
 616           const load_balance_t *lb0;
 617           vlib_buffer_t *p0;
 618           u32 pi0, lbi0, hc0;
 619           const ip4_header_t *ip0;
 620           const dpo_id_t *dpo0;
 621
 622           pi0 = from[0];
 623           to_next[0] = pi0;
 624           from += 1;
 625           to_next += 1;
 626           n_left_to_next -= 1;
 627           n_left_from -= 1;
 628
 629           p0 = vlib_get_buffer (vm, pi0);
 630
 631           ip0 = vlib_buffer_get_current (p0);
 632           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 633
 634           lb0 = load_balance_get (lbi0);
 635
 636           hc0 = 0;
 637           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 638             {
 639               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 640                 {
 641                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 642                     vnet_buffer (p0)->ip.flow_hash >> 1;
 643                 }
 644               else
 645                 {
 646                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 647                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 648                 }
 649               dpo0 = load_balance_get_fwd_bucket
 650                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 651             }
 652           else
 653             {
 654               dpo0 = load_balance_get_bucket_i (lb0, 0);
 655             }
 656
 657           next0 = dpo0->dpoi_next_node;
 658           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 659
 660           vlib_increment_combined_counter
 661             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 662
 663           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 664                                            to_next, n_left_to_next,
 665                                            pi0, next0);
 666         }
 667
 668       vlib_put_next_frame (vm, node, next, n_left_to_next);
 669     }
 670
 671   return frame->n_vectors;
 672 }
 673
 674 VLIB_REGISTER_NODE (ip4_load_balance_node) =
 675 {
 676 .function = ip4_load_balance,.name = "ip4-load-balance",.vector_size =
 677     sizeof (u32),.sibling_of = "ip4-lookup",.format_trace =
 678     format_ip4_lookup_trace,};
 679
 680 VLIB_NODE_FUNCTION_MULTIARCH (ip4_load_balance_node, ip4_load_balance);
 681
 682 /* get first interface address */
 683 ip4_address_t *
 684 ip4_interface_first_address (ip4_main_t * im, u32 sw_if_index,
 685                              ip_interface_address_t ** result_ia)
 686 {
 687   ip_lookup_main_t *lm = &im->lookup_main;
 688   ip_interface_address_t *ia = 0;
 689   ip4_address_t *result = 0;
 690
 691   /* *INDENT-OFF* */
 692   foreach_ip_interface_address
 693     (lm, ia, sw_if_index,
 694      1 /* honor unnumbered */ ,
 695      ({
 696        ip4_address_t * a =
 697          ip_interface_address_get_address (lm, ia);
 698        result = a;
 699        break;
 700      }));
 701   /* *INDENT-OFF* */
 702   if (result_ia)
 703     *result_ia = result ? ia : 0;
 704   return result;
 705 }
 706
 707 static void
 708 ip4_add_interface_routes (u32 sw_if_index,
 709                           ip4_main_t * im, u32 fib_index,
 710                           ip_interface_address_t * a)
 711 {
 712   ip_lookup_main_t *lm = &im->lookup_main;
 713   ip4_address_t *address = ip_interface_address_get_address (lm, a);
 714   fib_prefix_t pfx = {
 715     .fp_len = a->address_length,
 716     .fp_proto = FIB_PROTOCOL_IP4,
 717     .fp_addr.ip4 = *address,
 718   };
 719
 720   if (pfx.fp_len <= 30)
 721     {
 722       /* a /30 or shorter - add a glean for the network address */
 723       fib_table_entry_update_one_path (fib_index, &pfx,
 724                                        FIB_SOURCE_INTERFACE,
 725                                        (FIB_ENTRY_FLAG_CONNECTED |
 726                                         FIB_ENTRY_FLAG_ATTACHED),
 727                                        DPO_PROTO_IP4,
 728                                        /* No next-hop address */
 729                                        NULL,
 730                                        sw_if_index,
 731                                        // invalid FIB index
 732                                        ~0,
 733                                        1,
 734                                        // no out-label stack
 735                                        NULL,
 736                                        FIB_ROUTE_PATH_FLAG_NONE);
 737
 738       /* Add the two broadcast addresses as drop */
 739       fib_prefix_t net_pfx = {
 740         .fp_len = 32,
 741         .fp_proto = FIB_PROTOCOL_IP4,
 742         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 743       };
 744       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 745         fib_table_entry_special_add(fib_index,
 746                                     &net_pfx,
 747                                     FIB_SOURCE_INTERFACE,
 748                                     (FIB_ENTRY_FLAG_DROP |
 749                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 750       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 751       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 752         fib_table_entry_special_add(fib_index,
 753                                     &net_pfx,
 754                                     FIB_SOURCE_INTERFACE,
 755                                     (FIB_ENTRY_FLAG_DROP |
 756                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 757     }
 758   else if (pfx.fp_len == 31)
 759     {
 760       u32 mask = clib_host_to_net_u32(1);
 761       fib_prefix_t net_pfx = pfx;
 762
 763       net_pfx.fp_len = 32;
 764       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 765
 766       /* a /31 - add the other end as an attached host */
 767       fib_table_entry_update_one_path (fib_index, &net_pfx,
 768                                        FIB_SOURCE_INTERFACE,
 769                                        (FIB_ENTRY_FLAG_ATTACHED),
 770                                        DPO_PROTO_IP4,
 771                                        &net_pfx.fp_addr,
 772                                        sw_if_index,
 773                                        // invalid FIB index
 774                                        ~0,
 775                                        1,
 776                                        NULL,
 777                                        FIB_ROUTE_PATH_FLAG_NONE);
 778     }
 779   pfx.fp_len = 32;
 780
 781   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 782     {
 783       u32 classify_table_index =
 784         lm->classify_table_index_by_sw_if_index[sw_if_index];
 785       if (classify_table_index != (u32) ~ 0)
 786         {
 787           dpo_id_t dpo = DPO_INVALID;
 788
 789           dpo_set (&dpo,
 790                    DPO_CLASSIFY,
 791                    DPO_PROTO_IP4,
 792                    classify_dpo_create (DPO_PROTO_IP4, classify_table_index));
 793
 794           fib_table_entry_special_dpo_add (fib_index,
 795                                            &pfx,
 796                                            FIB_SOURCE_CLASSIFY,
 797                                            FIB_ENTRY_FLAG_NONE, &dpo);
 798           dpo_reset (&dpo);
 799         }
 800     }
 801
 802   fib_table_entry_update_one_path (fib_index, &pfx,
 803                                    FIB_SOURCE_INTERFACE,
 804                                    (FIB_ENTRY_FLAG_CONNECTED |
 805                                     FIB_ENTRY_FLAG_LOCAL),
 806                                    DPO_PROTO_IP4,
 807                                    &pfx.fp_addr,
 808                                    sw_if_index,
 809                                    // invalid FIB index
 810                                    ~0,
 811                                    1, NULL,
 812                                    FIB_ROUTE_PATH_FLAG_NONE);
 813 }
 814
 815 static void
 816 ip4_del_interface_routes (ip4_main_t * im,
 817                           u32 fib_index,
 818                           ip4_address_t * address, u32 address_length)
 819 {
 820   fib_prefix_t pfx = {
 821     .fp_len = address_length,
 822     .fp_proto = FIB_PROTOCOL_IP4,
 823     .fp_addr.ip4 = *address,
 824   };
 825
 826   if (pfx.fp_len <= 30)
 827     {
 828       fib_prefix_t net_pfx = {
 829         .fp_len = 32,
 830         .fp_proto = FIB_PROTOCOL_IP4,
 831         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 832       };
 833       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 834         fib_table_entry_special_remove(fib_index,
 835                                        &net_pfx,
 836                                        FIB_SOURCE_INTERFACE);
 837       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 838       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 839         fib_table_entry_special_remove(fib_index,
 840                                        &net_pfx,
 841                                        FIB_SOURCE_INTERFACE);
 842       fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 843     }
 844     else if (pfx.fp_len == 31)
 845     {
 846       u32 mask = clib_host_to_net_u32(1);
 847       fib_prefix_t net_pfx = pfx;
 848
 849       net_pfx.fp_len = 32;
 850       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 851
 852       fib_table_entry_delete (fib_index, &net_pfx, FIB_SOURCE_INTERFACE);
 853     }
 854
 855   pfx.fp_len = 32;
 856   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 857 }
 858
 859 void
 860 ip4_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 861 {
 862   ip4_main_t *im = &ip4_main;
 863
 864   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 865
 866   /*
 867    * enable/disable only on the 1<->0 transition
 868    */
 869   if (is_enable)
 870     {
 871       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 872         return;
 873     }
 874   else
 875     {
 876       ASSERT (im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 877       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 878         return;
 879     }
 880   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
 881                                !is_enable, 0, 0);
 882
 883
 884   vnet_feature_enable_disable ("ip4-multicast", "ip4-drop",
 885                                sw_if_index, !is_enable, 0, 0);
 886 }
 887
 888 static clib_error_t *
 889 ip4_add_del_interface_address_internal (vlib_main_t * vm,
 890                                         u32 sw_if_index,
 891                                         ip4_address_t * address,
 892                                         u32 address_length, u32 is_del)
 893 {
 894   vnet_main_t *vnm = vnet_get_main ();
 895   ip4_main_t *im = &ip4_main;
 896   ip_lookup_main_t *lm = &im->lookup_main;
 897   clib_error_t *error = 0;
 898   u32 if_address_index, elts_before;
 899   ip4_address_fib_t ip4_af, *addr_fib = 0;
 900
 901   /* local0 interface doesn't support IP addressing  */
 902   if (sw_if_index == 0)
 903     {
 904       return
 905        clib_error_create ("local0 interface doesn't support IP addressing");
 906     }
 907
 908   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 909   ip4_addr_fib_init (&ip4_af, address,
 910                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 911   vec_add1 (addr_fib, ip4_af);
 912
 913   /* FIXME-LATER
 914    * there is no support for adj-fib handling in the presence of overlapping
 915    * subnets on interfaces. Easy fix - disallow overlapping subnets, like
 916    * most routers do.
 917    */
 918   /* *INDENT-OFF* */
 919   if (!is_del)
 920     {
 921       /* When adding an address check that it does not conflict
 922          with an existing address. */
 923       ip_interface_address_t *ia;
 924       foreach_ip_interface_address
 925         (&im->lookup_main, ia, sw_if_index,
 926          0 /* honor unnumbered */ ,
 927          ({
 928            ip4_address_t * x =
 929              ip_interface_address_get_address
 930              (&im->lookup_main, ia);
 931            if (ip4_destination_matches_route
 932                (im, address, x, ia->address_length) ||
 933                ip4_destination_matches_route (im,
 934                                               x,
 935                                               address,
 936                                               address_length))
 937              return
 938                clib_error_create
 939                ("failed to add %U which conflicts with %U for interface %U",
 940                 format_ip4_address_and_length, address,
 941                 address_length,
 942                 format_ip4_address_and_length, x,
 943                 ia->address_length,
 944                 format_vnet_sw_if_index_name, vnm,
 945                 sw_if_index);
 946          }));
 947     }
 948   /* *INDENT-ON* */
 949
 950   elts_before = pool_elts (lm->if_address_pool);
 951
 952   error = ip_interface_address_add_del
 953     (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 954   if (error)
 955     goto done;
 956
 957   ip4_sw_interface_enable_disable (sw_if_index, !is_del);
 958
 959   if (is_del)
 960     ip4_del_interface_routes (im, ip4_af.fib_index, address, address_length);
 961   else
 962     ip4_add_interface_routes (sw_if_index,
 963                               im, ip4_af.fib_index,
 964                               pool_elt_at_index
 965                               (lm->if_address_pool, if_address_index));
 966
 967   /* If pool did not grow/shrink: add duplicate address. */
 968   if (elts_before != pool_elts (lm->if_address_pool))
 969     {
 970       ip4_add_del_interface_address_callback_t *cb;
 971       vec_foreach (cb, im->add_del_interface_address_callbacks)
 972         cb->function (im, cb->function_opaque, sw_if_index,
 973                       address, address_length, if_address_index, is_del);
 974     }
 975
 976 done:
 977   vec_free (addr_fib);
 978   return error;
 979 }
 980
 981 clib_error_t *
 982 ip4_add_del_interface_address (vlib_main_t * vm,
 983                                u32 sw_if_index,
 984                                ip4_address_t * address,
 985                                u32 address_length, u32 is_del)
 986 {
 987   return ip4_add_del_interface_address_internal
 988     (vm, sw_if_index, address, address_length, is_del);
 989 }
 990
 991 /* Built-in ip4 unicast rx feature path definition */
 992 /* *INDENT-OFF* */
 993 VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
 994 {
 995   .arc_name = "ip4-unicast",
 996   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 997   .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
 998 };
 999
1000 VNET_FEATURE_INIT (ip4_flow_classify, static) =
1001 {
1002   .arc_name = "ip4-unicast",
1003   .node_name = "ip4-flow-classify",
1004   .runs_before = VNET_FEATURES ("ip4-inacl"),
1005 };
1006
1007 VNET_FEATURE_INIT (ip4_inacl, static) =
1008 {
1009   .arc_name = "ip4-unicast",
1010   .node_name = "ip4-inacl",
1011   .runs_before = VNET_FEATURES ("ip4-source-check-via-rx"),
1012 };
1013
1014 VNET_FEATURE_INIT (ip4_source_check_1, static) =
1015 {
1016   .arc_name = "ip4-unicast",
1017   .node_name = "ip4-source-check-via-rx",
1018   .runs_before = VNET_FEATURES ("ip4-source-check-via-any"),
1019 };
1020
1021 VNET_FEATURE_INIT (ip4_source_check_2, static) =
1022 {
1023   .arc_name = "ip4-unicast",
1024   .node_name = "ip4-source-check-via-any",
1025   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
1026 };
1027
1028 VNET_FEATURE_INIT (ip4_source_and_port_range_check_rx, static) =
1029 {
1030   .arc_name = "ip4-unicast",
1031   .node_name = "ip4-source-and-port-range-check-rx",
1032   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
1033 };
1034
1035 VNET_FEATURE_INIT (ip4_policer_classify, static) =
1036 {
1037   .arc_name = "ip4-unicast",
1038   .node_name = "ip4-policer-classify",
1039   .runs_before = VNET_FEATURES ("ipsec-input-ip4"),
1040 };
1041
1042 VNET_FEATURE_INIT (ip4_ipsec, static) =
1043 {
1044   .arc_name = "ip4-unicast",
1045   .node_name = "ipsec-input-ip4",
1046   .runs_before = VNET_FEATURES ("vpath-input-ip4"),
1047 };
1048
1049 VNET_FEATURE_INIT (ip4_vpath, static) =
1050 {
1051   .arc_name = "ip4-unicast",
1052   .node_name = "vpath-input-ip4",
1053   .runs_before = VNET_FEATURES ("ip4-vxlan-bypass"),
1054 };
1055
1056 VNET_FEATURE_INIT (ip4_vxlan_bypass, static) =
1057 {
1058   .arc_name = "ip4-unicast",
1059   .node_name = "ip4-vxlan-bypass",
1060   .runs_before = VNET_FEATURES ("ip4-lookup"),
1061 };
1062
1063 VNET_FEATURE_INIT (ip4_drop, static) =
1064 {
1065   .arc_name = "ip4-unicast",
1066   .node_name = "ip4-drop",
1067   .runs_before = VNET_FEATURES ("ip4-lookup"),
1068 };
1069
1070 VNET_FEATURE_INIT (ip4_lookup, static) =
1071 {
1072   .arc_name = "ip4-unicast",
1073   .node_name = "ip4-lookup",
1074   .runs_before = 0,     /* not before any other features */
1075 };
1076
1077 /* Built-in ip4 multicast rx feature path definition */
1078 VNET_FEATURE_ARC_INIT (ip4_multicast, static) =
1079 {
1080   .arc_name = "ip4-multicast",
1081   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
1082   .arc_index_ptr = &ip4_main.lookup_main.mcast_feature_arc_index,
1083 };
1084
1085 VNET_FEATURE_INIT (ip4_vpath_mc, static) =
1086 {
1087   .arc_name = "ip4-multicast",
1088   .node_name = "vpath-input-ip4",
1089   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
1090 };
1091
1092 VNET_FEATURE_INIT (ip4_mc_drop, static) =
1093 {
1094   .arc_name = "ip4-multicast",
1095   .node_name = "ip4-drop",
1096   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
1097 };
1098
1099 VNET_FEATURE_INIT (ip4_lookup_mc, static) =
1100 {
1101   .arc_name = "ip4-multicast",
1102   .node_name = "ip4-mfib-forward-lookup",
1103   .runs_before = 0,     /* last feature */
1104 };
1105
1106 /* Source and port-range check ip4 tx feature path definition */
1107 VNET_FEATURE_ARC_INIT (ip4_output, static) =
1108 {
1109   .arc_name = "ip4-output",
1110   .start_nodes = VNET_FEATURES ("ip4-rewrite", "ip4-midchain"),
1111   .arc_index_ptr = &ip4_main.lookup_main.output_feature_arc_index,
1112 };
1113
1114 VNET_FEATURE_INIT (ip4_source_and_port_range_check_tx, static) =
1115 {
1116   .arc_name = "ip4-output",
1117   .node_name = "ip4-source-and-port-range-check-tx",
1118   .runs_before = VNET_FEATURES ("ipsec-output-ip4"),
1119 };
1120
1121 VNET_FEATURE_INIT (ip4_ipsec_output, static) =
1122 {
1123   .arc_name = "ip4-output",
1124   .node_name = "ipsec-output-ip4",
1125   .runs_before = VNET_FEATURES ("interface-output"),
1126 };
1127
1128 /* Built-in ip4 tx feature path definition */
1129 VNET_FEATURE_INIT (ip4_interface_output, static) =
1130 {
1131   .arc_name = "ip4-output",
1132   .node_name = "interface-output",
1133   .runs_before = 0,     /* not before any other features */
1134 };
1135 /* *INDENT-ON* */
1136
1137 static clib_error_t *
1138 ip4_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
1139 {
1140   ip4_main_t *im = &ip4_main;
1141
1142   /* Fill in lookup tables with default table (0). */
1143   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1144   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
1145
1146   if (!is_add)
1147     {
1148       ip4_main_t *im4 = &ip4_main;
1149       ip_lookup_main_t *lm4 = &im4->lookup_main;
1150       ip_interface_address_t *ia = 0;
1151       ip4_address_t *address;
1152       vlib_main_t *vm = vlib_get_main ();
1153
1154       /* *INDENT-OFF* */
1155       foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* honor unnumbered */,
1156       ({
1157         address = ip_interface_address_get_address (lm4, ia);
1158         ip4_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
1159       }));
1160       /* *INDENT-ON* */
1161     }
1162
1163   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
1164                                is_add, 0, 0);
1165
1166   vnet_feature_enable_disable ("ip4-multicast", "ip4-drop", sw_if_index,
1167                                is_add, 0, 0);
1168
1169   return /* no error */ 0;
1170 }
1171
1172 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
1173
1174 /* Global IP4 main. */
1175 ip4_main_t ip4_main;
1176
1177 clib_error_t *
1178 ip4_lookup_init (vlib_main_t * vm)
1179 {
1180   ip4_main_t *im = &ip4_main;
1181   clib_error_t *error;
1182   uword i;
1183
1184   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
1185     return error;
1186
1187   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
1188     {
1189       u32 m;
1190
1191       if (i < 32)
1192         m = pow2_mask (i) << (32 - i);
1193       else
1194         m = ~0;
1195       im->fib_masks[i] = clib_host_to_net_u32 (m);
1196     }
1197
1198   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
1199
1200   /* Create FIB with index 0 and table id of 0. */
1201   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0);
1202   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0);
1203
1204   {
1205     pg_node_t *pn;
1206     pn = pg_get_node (ip4_lookup_node.index);
1207     pn->unformat_edit = unformat_pg_ip4_header;
1208   }
1209
1210   {
1211     ethernet_arp_header_t h;
1212
1213     memset (&h, 0, sizeof (h));
1214
1215     /* Set target ethernet address to all zeros. */
1216     memset (h.ip4_over_ethernet[1].ethernet, 0,
1217             sizeof (h.ip4_over_ethernet[1].ethernet));
1218
1219 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
1220 #define _8(f,v) h.f = v;
1221     _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
1222     _16 (l3_type, ETHERNET_TYPE_IP4);
1223     _8 (n_l2_address_bytes, 6);
1224     _8 (n_l3_address_bytes, 4);
1225     _16 (opcode, ETHERNET_ARP_OPCODE_request);
1226 #undef _16
1227 #undef _8
1228
1229     vlib_packet_template_init (vm, &im->ip4_arp_request_packet_template,
1230                                /* data */ &h,
1231                                sizeof (h),
1232                                /* alloc chunk size */ 8,
1233                                "ip4 arp");
1234   }
1235
1236   return error;
1237 }
1238
1239 VLIB_INIT_FUNCTION (ip4_lookup_init);
1240
1241 typedef struct
1242 {
1243   /* Adjacency taken. */
1244   u32 dpo_index;
1245   u32 flow_hash;
1246   u32 fib_index;
1247
1248   /* Packet data, possibly *after* rewrite. */
1249   u8 packet_data[64 - 1 * sizeof (u32)];
1250 }
1251 ip4_forward_next_trace_t;
1252
1253 u8 *
1254 format_ip4_forward_next_trace (u8 * s, va_list * args)
1255 {
1256   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1257   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1258   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1259   uword indent = format_get_indent (s);
1260   s = format (s, "%U%U",
1261               format_white_space, indent,
1262               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1263   return s;
1264 }
1265
1266 static u8 *
1267 format_ip4_lookup_trace (u8 * s, va_list * args)
1268 {
1269   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1270   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1271   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1272   uword indent = format_get_indent (s);
1273
1274   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
1275               t->fib_index, t->dpo_index, t->flow_hash);
1276   s = format (s, "\n%U%U",
1277               format_white_space, indent,
1278               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1279   return s;
1280 }
1281
1282 static u8 *
1283 format_ip4_rewrite_trace (u8 * s, va_list * args)
1284 {
1285   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1286   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1287   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1288   uword indent = format_get_indent (s);
1289
1290   s = format (s, "tx_sw_if_index %d dpo-idx %d : %U flow hash: 0x%08x",
1291               t->fib_index, t->dpo_index, format_ip_adjacency,
1292               t->dpo_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
1293   s = format (s, "\n%U%U",
1294               format_white_space, indent,
1295               format_ip_adjacency_packet_data,
1296               t->dpo_index, t->packet_data, sizeof (t->packet_data));
1297   return s;
1298 }
1299
1300 /* Common trace function for all ip4-forward next nodes. */
1301 void
1302 ip4_forward_next_trace (vlib_main_t * vm,
1303                         vlib_node_runtime_t * node,
1304                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
1305 {
1306   u32 *from, n_left;
1307   ip4_main_t *im = &ip4_main;
1308
1309   n_left = frame->n_vectors;
1310   from = vlib_frame_vector_args (frame);
1311
1312   while (n_left >= 4)
1313     {
1314       u32 bi0, bi1;
1315       vlib_buffer_t *b0, *b1;
1316       ip4_forward_next_trace_t *t0, *t1;
1317
1318       /* Prefetch next iteration. */
1319       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1320       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1321
1322       bi0 = from[0];
1323       bi1 = from[1];
1324
1325       b0 = vlib_get_buffer (vm, bi0);
1326       b1 = vlib_get_buffer (vm, bi1);
1327
1328       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1329         {
1330           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1331           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1332           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1333           t0->fib_index =
1334             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1335              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1336             vec_elt (im->fib_index_by_sw_if_index,
1337                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1338
1339           clib_memcpy (t0->packet_data,
1340                        vlib_buffer_get_current (b0),
1341                        sizeof (t0->packet_data));
1342         }
1343       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1344         {
1345           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1346           t1->dpo_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1347           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1348           t1->fib_index =
1349             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
1350              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
1351             vec_elt (im->fib_index_by_sw_if_index,
1352                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
1353           clib_memcpy (t1->packet_data, vlib_buffer_get_current (b1),
1354                        sizeof (t1->packet_data));
1355         }
1356       from += 2;
1357       n_left -= 2;
1358     }
1359
1360   while (n_left >= 1)
1361     {
1362       u32 bi0;
1363       vlib_buffer_t *b0;
1364       ip4_forward_next_trace_t *t0;
1365
1366       bi0 = from[0];
1367
1368       b0 = vlib_get_buffer (vm, bi0);
1369
1370       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1371         {
1372           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1373           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1374           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1375           t0->fib_index =
1376             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1377              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1378             vec_elt (im->fib_index_by_sw_if_index,
1379                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1380           clib_memcpy (t0->packet_data, vlib_buffer_get_current (b0),
1381                        sizeof (t0->packet_data));
1382         }
1383       from += 1;
1384       n_left -= 1;
1385     }
1386 }
1387
1388 static uword
1389 ip4_drop_or_punt (vlib_main_t * vm,
1390                   vlib_node_runtime_t * node,
1391                   vlib_frame_t * frame, ip4_error_t error_code)
1392 {
1393   u32 *buffers = vlib_frame_vector_args (frame);
1394   uword n_packets = frame->n_vectors;
1395
1396   vlib_error_drop_buffers (vm, node, buffers,
1397                            /* stride */ 1,
1398                            n_packets,
1399                            /* next */ 0,
1400                            ip4_input_node.index, error_code);
1401
1402   if (node->flags & VLIB_NODE_FLAG_TRACE)
1403     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1404
1405   return n_packets;
1406 }
1407
1408 static uword
1409 ip4_drop (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1410 {
1411   return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_DROP);
1412 }
1413
1414 static uword
1415 ip4_punt (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1416 {
1417   return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_PUNT);
1418 }
1419
1420 /* *INDENT-OFF* */
1421 VLIB_REGISTER_NODE (ip4_drop_node, static) =
1422 {
1423   .function = ip4_drop,
1424   .name = "ip4-drop",
1425   .vector_size = sizeof (u32),
1426   .format_trace = format_ip4_forward_next_trace,
1427   .n_next_nodes = 1,
1428   .next_nodes = {
1429     [0] = "error-drop",
1430   },
1431 };
1432
1433 VLIB_NODE_FUNCTION_MULTIARCH (ip4_drop_node, ip4_drop);
1434
1435 VLIB_REGISTER_NODE (ip4_punt_node, static) =
1436 {
1437   .function = ip4_punt,
1438   .name = "ip4-punt",
1439   .vector_size = sizeof (u32),
1440   .format_trace = format_ip4_forward_next_trace,
1441   .n_next_nodes = 1,
1442   .next_nodes = {
1443     [0] = "error-punt",
1444   },
1445 };
1446
1447 VLIB_NODE_FUNCTION_MULTIARCH (ip4_punt_node, ip4_punt);
1448 /* *INDENT-ON */
1449
1450 /* Compute TCP/UDP/ICMP4 checksum in software. */
1451 u16
1452 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1453                               ip4_header_t * ip0)
1454 {
1455   ip_csum_t sum0;
1456   u32 ip_header_length, payload_length_host_byte_order;
1457   u32 n_this_buffer, n_bytes_left, n_ip_bytes_this_buffer;
1458   u16 sum16;
1459   void *data_this_buffer;
1460
1461   /* Initialize checksum with ip header. */
1462   ip_header_length = ip4_header_bytes (ip0);
1463   payload_length_host_byte_order =
1464     clib_net_to_host_u16 (ip0->length) - ip_header_length;
1465   sum0 =
1466     clib_host_to_net_u32 (payload_length_host_byte_order +
1467                           (ip0->protocol << 16));
1468
1469   if (BITS (uword) == 32)
1470     {
1471       sum0 =
1472         ip_csum_with_carry (sum0,
1473                             clib_mem_unaligned (&ip0->src_address, u32));
1474       sum0 =
1475         ip_csum_with_carry (sum0,
1476                             clib_mem_unaligned (&ip0->dst_address, u32));
1477     }
1478   else
1479     sum0 =
1480       ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1481
1482   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1483   data_this_buffer = (void *) ip0 + ip_header_length;
1484   n_ip_bytes_this_buffer = p0->current_length - (((u8 *) ip0 - p0->data) - p0->current_data);
1485   if (n_this_buffer + ip_header_length > n_ip_bytes_this_buffer)
1486     {
1487       n_this_buffer = n_ip_bytes_this_buffer > ip_header_length ?
1488           n_ip_bytes_this_buffer - ip_header_length : 0;
1489     }
1490   while (1)
1491     {
1492       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1493       n_bytes_left -= n_this_buffer;
1494       if (n_bytes_left == 0)
1495         break;
1496
1497       ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1498       p0 = vlib_get_buffer (vm, p0->next_buffer);
1499       data_this_buffer = vlib_buffer_get_current (p0);
1500       n_this_buffer = p0->current_length;
1501     }
1502
1503   sum16 = ~ip_csum_fold (sum0);
1504
1505   return sum16;
1506 }
1507
1508 u32
1509 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1510 {
1511   ip4_header_t *ip0 = vlib_buffer_get_current (p0);
1512   udp_header_t *udp0;
1513   u16 sum16;
1514
1515   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1516           || ip0->protocol == IP_PROTOCOL_UDP);
1517
1518   udp0 = (void *) (ip0 + 1);
1519   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1520     {
1521       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1522                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
1523       return p0->flags;
1524     }
1525
1526   sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1527
1528   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1529                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
1530
1531   return p0->flags;
1532 }
1533
1534 /* *INDENT-OFF* */
1535 VNET_FEATURE_ARC_INIT (ip4_local) =
1536 {
1537   .arc_name  = "ip4-local",
1538   .start_nodes = VNET_FEATURES ("ip4-local"),
1539 };
1540 /* *INDENT-ON* */
1541
1542 static inline void
1543 ip4_local_validate_l4 (vlib_main_t * vm, vlib_buffer_t * p, ip4_header_t * ip,
1544                        u8 is_udp, u8 * error, u8 * good_tcp_udp)
1545 {
1546   u32 flags0;
1547   flags0 = ip4_tcp_udp_validate_checksum (vm, p);
1548   *good_tcp_udp = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1549   if (is_udp)
1550     {
1551       udp_header_t *udp;
1552       u32 ip_len, udp_len;
1553       i32 len_diff;
1554       udp = ip4_next_header (ip);
1555       /* Verify UDP length. */
1556       ip_len = clib_net_to_host_u16 (ip->length);
1557       udp_len = clib_net_to_host_u16 (udp->length);
1558
1559       len_diff = ip_len - udp_len;
1560       *good_tcp_udp &= len_diff >= 0;
1561       *error = len_diff < 0 ? IP4_ERROR_UDP_LENGTH : *error;
1562     }
1563 }
1564
1565 #define ip4_local_do_l4_check(is_tcp_udp, flags)                        \
1566     (is_tcp_udp && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED))
1567
1568 static inline uword
1569 ip4_local_inline (vlib_main_t * vm,
1570                   vlib_node_runtime_t * node,
1571                   vlib_frame_t * frame, int head_of_feature_arc)
1572 {
1573   ip4_main_t *im = &ip4_main;
1574   ip_lookup_main_t *lm = &im->lookup_main;
1575   ip_local_next_t next_index;
1576   u32 *from, *to_next, n_left_from, n_left_to_next;
1577   vlib_node_runtime_t *error_node =
1578     vlib_node_get_runtime (vm, ip4_input_node.index);
1579   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1580
1581   from = vlib_frame_vector_args (frame);
1582   n_left_from = frame->n_vectors;
1583   next_index = node->cached_next_index;
1584
1585   if (node->flags & VLIB_NODE_FLAG_TRACE)
1586     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1587
1588   while (n_left_from > 0)
1589     {
1590       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1591
1592       while (n_left_from >= 4 && n_left_to_next >= 2)
1593         {
1594           vlib_buffer_t *p0, *p1;
1595           ip4_header_t *ip0, *ip1;
1596           ip4_fib_mtrie_t *mtrie0, *mtrie1;
1597           ip4_fib_mtrie_leaf_t leaf0, leaf1;
1598           const dpo_id_t *dpo0, *dpo1;
1599           const load_balance_t *lb0, *lb1;
1600           u32 pi0, next0, fib_index0, lbi0;
1601           u32 pi1, next1, fib_index1, lbi1;
1602           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1603           u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1604           u32 sw_if_index0, sw_if_index1;
1605
1606           pi0 = to_next[0] = from[0];
1607           pi1 = to_next[1] = from[1];
1608           from += 2;
1609           n_left_from -= 2;
1610           to_next += 2;
1611           n_left_to_next -= 2;
1612
1613           next0 = next1 = IP_LOCAL_NEXT_DROP;
1614           error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1615
1616           p0 = vlib_get_buffer (vm, pi0);
1617           p1 = vlib_get_buffer (vm, pi1);
1618
1619           ip0 = vlib_buffer_get_current (p0);
1620           ip1 = vlib_buffer_get_current (p1);
1621
1622           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1623           vnet_buffer (p1)->l3_hdr_offset = p1->current_data;
1624
1625           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1626           sw_if_index1 = vnet_buffer (p1)->sw_if_index[VLIB_RX];
1627
1628           /* Treat IP frag packets as "experimental" protocol for now
1629              until support of IP frag reassembly is implemented */
1630           proto0 = ip4_is_fragment (ip0) ? 0xfe : ip0->protocol;
1631           proto1 = ip4_is_fragment (ip1) ? 0xfe : ip1->protocol;
1632
1633           if (head_of_feature_arc == 0)
1634             goto skip_checks;
1635
1636           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1637           is_udp1 = proto1 == IP_PROTOCOL_UDP;
1638           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1639           is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1640
1641           good_tcp_udp0 =
1642             (p0->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1643           good_tcp_udp1 =
1644             (p1->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1645
1646           if (PREDICT_FALSE (ip4_local_do_l4_check (is_tcp_udp0, p0->flags)
1647                              || ip4_local_do_l4_check (is_tcp_udp1,
1648                                                        p1->flags)))
1649             {
1650               if (is_tcp_udp0)
1651                 ip4_local_validate_l4 (vm, p0, ip0, is_udp0, &error0,
1652                                        &good_tcp_udp0);
1653               if (is_tcp_udp1)
1654                 ip4_local_validate_l4 (vm, p1, ip1, is_udp1, &error1,
1655                                        &good_tcp_udp1);
1656             }
1657
1658           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1659           error0 = (is_tcp_udp0 && !good_tcp_udp0
1660                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1661           error1 = (is_tcp_udp1 && !good_tcp_udp1
1662                     ? IP4_ERROR_TCP_CHECKSUM + is_udp1 : error1);
1663
1664           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1665           fib_index0 =
1666             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1667              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1668
1669           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index1);
1670           fib_index1 =
1671             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
1672              (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
1673
1674           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1675           mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
1676
1677           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1678           leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, &ip1->src_address);
1679           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1680                                              2);
1681           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address,
1682                                              2);
1683           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1684                                              3);
1685           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address,
1686                                              3);
1687
1688           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0 =
1689             ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1690           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1691
1692           vnet_buffer (p1)->ip.adj_index[VLIB_RX] = lbi1 =
1693             ip4_fib_mtrie_leaf_get_adj_index (leaf1);
1694           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = lbi1;
1695
1696           lb0 = load_balance_get (lbi0);
1697           lb1 = load_balance_get (lbi1);
1698           dpo0 = load_balance_get_bucket_i (lb0, 0);
1699           dpo1 = load_balance_get_bucket_i (lb1, 0);
1700
1701           /*
1702            * Must have a route to source otherwise we drop the packet.
1703            * ip4 broadcasts are accepted, e.g. to make dhcp client work
1704            *
1705            * The checks are:
1706            *  - the source is a recieve => it's from us => bogus, do this
1707            *    first since it sets a different error code.
1708            *  - uRPF check for any route to source - accept if passes.
1709            *  - allow packets destined to the broadcast address from unknown sources
1710            */
1711           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1712                      dpo0->dpoi_type == DPO_RECEIVE) ?
1713                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1714           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1715                      !fib_urpf_check_size (lb0->lb_urpf) &&
1716                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1717                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1718           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1719                      dpo1->dpoi_type == DPO_RECEIVE) ?
1720                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error1);
1721           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1722                      !fib_urpf_check_size (lb1->lb_urpf) &&
1723                      ip1->dst_address.as_u32 != 0xFFFFFFFF)
1724                     ? IP4_ERROR_SRC_LOOKUP_MISS : error1);
1725
1726         skip_checks:
1727
1728           next0 = lm->local_next_by_ip_protocol[proto0];
1729           next1 = lm->local_next_by_ip_protocol[proto1];
1730
1731           next0 =
1732             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1733           next1 =
1734             error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1735
1736           p0->error = error0 ? error_node->errors[error0] : 0;
1737           p1->error = error1 ? error_node->errors[error1] : 0;
1738
1739           if (head_of_feature_arc)
1740             {
1741               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1742                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1743               if (PREDICT_TRUE (error1 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1744                 vnet_feature_arc_start (arc_index, sw_if_index1, &next1, p1);
1745             }
1746
1747           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1748                                            n_left_to_next, pi0, pi1,
1749                                            next0, next1);
1750         }
1751
1752       while (n_left_from > 0 && n_left_to_next > 0)
1753         {
1754           vlib_buffer_t *p0;
1755           ip4_header_t *ip0;
1756           ip4_fib_mtrie_t *mtrie0;
1757           ip4_fib_mtrie_leaf_t leaf0;
1758           u32 pi0, next0, fib_index0, lbi0;
1759           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1760           load_balance_t *lb0;
1761           const dpo_id_t *dpo0;
1762           u32 sw_if_index0;
1763
1764           pi0 = to_next[0] = from[0];
1765           from += 1;
1766           n_left_from -= 1;
1767           to_next += 1;
1768           n_left_to_next -= 1;
1769
1770           next0 = IP_LOCAL_NEXT_DROP;
1771           error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
1772
1773           p0 = vlib_get_buffer (vm, pi0);
1774           ip0 = vlib_buffer_get_current (p0);
1775           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1776           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1777
1778           /* Treat IP frag packets as "experimental" protocol for now
1779              until support of IP frag reassembly is implemented */
1780           proto0 = ip4_is_fragment (ip0) ? 0xfe : ip0->protocol;
1781
1782           if (head_of_feature_arc == 0)
1783             goto skip_check;
1784
1785           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1786           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1787           good_tcp_udp0 =
1788             (p0->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1789
1790           if (PREDICT_FALSE (ip4_local_do_l4_check (is_tcp_udp0, p0->flags)))
1791             {
1792               ip4_local_validate_l4 (vm, p0, ip0, is_udp0, &error0,
1793                                      &good_tcp_udp0);
1794             }
1795
1796           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1797           error0 = (is_tcp_udp0 && !good_tcp_udp0
1798                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1799
1800           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1801           fib_index0 =
1802             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1803              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1804           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1805           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1806           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1807                                              2);
1808           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1809                                              3);
1810           lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1811           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1812           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0;
1813
1814           lb0 = load_balance_get (lbi0);
1815           dpo0 = load_balance_get_bucket_i (lb0, 0);
1816
1817           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1818                      dpo0->dpoi_type == DPO_RECEIVE) ?
1819                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1820           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1821                      !fib_urpf_check_size (lb0->lb_urpf) &&
1822                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1823                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1824
1825         skip_check:
1826           next0 = lm->local_next_by_ip_protocol[proto0];
1827           next0 =
1828             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1829
1830           p0->error = error0 ? error_node->errors[error0] : 0;
1831
1832           if (head_of_feature_arc)
1833             {
1834               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1835                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1836             }
1837
1838           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1839                                            n_left_to_next, pi0, next0);
1840         }
1841       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1842     }
1843
1844   return frame->n_vectors;
1845 }
1846
1847 static uword
1848 ip4_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1849 {
1850   return ip4_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1851 }
1852
1853 /* *INDENT-OFF* */
1854 VLIB_REGISTER_NODE (ip4_local_node) =
1855 {
1856   .function = ip4_local,
1857   .name = "ip4-local",
1858   .vector_size = sizeof (u32),
1859   .format_trace = format_ip4_forward_next_trace,
1860   .n_next_nodes = IP_LOCAL_N_NEXT,
1861   .next_nodes =
1862   {
1863     [IP_LOCAL_NEXT_DROP] = "error-drop",
1864     [IP_LOCAL_NEXT_PUNT] = "error-punt",
1865     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
1866     [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",
1867   },
1868 };
1869 /* *INDENT-ON* */
1870
1871 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local);
1872
1873 static uword
1874 ip4_local_end_of_arc (vlib_main_t * vm,
1875                       vlib_node_runtime_t * node, vlib_frame_t * frame)
1876 {
1877   return ip4_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1878 }
1879
1880 /* *INDENT-OFF* */
1881 VLIB_REGISTER_NODE (ip4_local_end_of_arc_node,static) = {
1882   .function = ip4_local_end_of_arc,
1883   .name = "ip4-local-end-of-arc",
1884   .vector_size = sizeof (u32),
1885
1886   .format_trace = format_ip4_forward_next_trace,
1887   .sibling_of = "ip4-local",
1888 };
1889
1890 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_end_of_arc_node, ip4_local_end_of_arc)
1891
1892 VNET_FEATURE_INIT (ip4_local_end_of_arc, static) = {
1893   .arc_name = "ip4-local",
1894   .node_name = "ip4-local-end-of-arc",
1895   .runs_before = 0, /* not before any other features */
1896 };
1897 /* *INDENT-ON* */
1898
1899 void
1900 ip4_register_protocol (u32 protocol, u32 node_index)
1901 {
1902   vlib_main_t *vm = vlib_get_main ();
1903   ip4_main_t *im = &ip4_main;
1904   ip_lookup_main_t *lm = &im->lookup_main;
1905
1906   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1907   lm->local_next_by_ip_protocol[protocol] =
1908     vlib_node_add_next (vm, ip4_local_node.index, node_index);
1909 }
1910
1911 static clib_error_t *
1912 show_ip_local_command_fn (vlib_main_t * vm,
1913                           unformat_input_t * input, vlib_cli_command_t * cmd)
1914 {
1915   ip4_main_t *im = &ip4_main;
1916   ip_lookup_main_t *lm = &im->lookup_main;
1917   int i;
1918
1919   vlib_cli_output (vm, "Protocols handled by ip4_local");
1920   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
1921     {
1922       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
1923         vlib_cli_output (vm, "%d", i);
1924     }
1925   return 0;
1926 }
1927
1928
1929
1930 /*?
1931  * Display the set of protocols handled by the local IPv4 stack.
1932  *
1933  * @cliexpar
1934  * Example of how to display local protocol table:
1935  * @cliexstart{show ip local}
1936  * Protocols handled by ip4_local
1937  * 1
1938  * 17
1939  * 47
1940  * @cliexend
1941 ?*/
1942 /* *INDENT-OFF* */
1943 VLIB_CLI_COMMAND (show_ip_local, static) =
1944 {
1945   .path = "show ip local",
1946   .function = show_ip_local_command_fn,
1947   .short_help = "show ip local",
1948 };
1949 /* *INDENT-ON* */
1950
1951 always_inline uword
1952 ip4_arp_inline (vlib_main_t * vm,
1953                 vlib_node_runtime_t * node,
1954                 vlib_frame_t * frame, int is_glean)
1955 {
1956   vnet_main_t *vnm = vnet_get_main ();
1957   ip4_main_t *im = &ip4_main;
1958   ip_lookup_main_t *lm = &im->lookup_main;
1959   u32 *from, *to_next_drop;
1960   uword n_left_from, n_left_to_next_drop, next_index;
1961   static f64 time_last_seed_change = -1e100;
1962   static u32 hash_seeds[3];
1963   static uword hash_bitmap[256 / BITS (uword)];
1964   f64 time_now;
1965
1966   if (node->flags & VLIB_NODE_FLAG_TRACE)
1967     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1968
1969   time_now = vlib_time_now (vm);
1970   if (time_now - time_last_seed_change > 1e-3)
1971     {
1972       uword i;
1973       u32 *r = clib_random_buffer_get_data (&vm->random_buffer,
1974                                             sizeof (hash_seeds));
1975       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1976         hash_seeds[i] = r[i];
1977
1978       /* Mark all hash keys as been no-seen before. */
1979       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1980         hash_bitmap[i] = 0;
1981
1982       time_last_seed_change = time_now;
1983     }
1984
1985   from = vlib_frame_vector_args (frame);
1986   n_left_from = frame->n_vectors;
1987   next_index = node->cached_next_index;
1988   if (next_index == IP4_ARP_NEXT_DROP)
1989     next_index = IP4_ARP_N_NEXT;        /* point to first interface */
1990
1991   while (n_left_from > 0)
1992     {
1993       vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
1994                            to_next_drop, n_left_to_next_drop);
1995
1996       while (n_left_from > 0 && n_left_to_next_drop > 0)
1997         {
1998           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
1999           ip_adjacency_t *adj0;
2000           vlib_buffer_t *p0;
2001           ip4_header_t *ip0;
2002           uword bm0;
2003
2004           pi0 = from[0];
2005
2006           p0 = vlib_get_buffer (vm, pi0);
2007
2008           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2009           adj0 = adj_get (adj_index0);
2010           ip0 = vlib_buffer_get_current (p0);
2011
2012           a0 = hash_seeds[0];
2013           b0 = hash_seeds[1];
2014           c0 = hash_seeds[2];
2015
2016           sw_if_index0 = adj0->rewrite_header.sw_if_index;
2017           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
2018
2019           if (is_glean)
2020             {
2021               /*
2022                * this is the Glean case, so we are ARPing for the
2023                * packet's destination
2024                */
2025               a0 ^= ip0->dst_address.data_u32;
2026             }
2027           else
2028             {
2029               a0 ^= adj0->sub_type.nbr.next_hop.ip4.data_u32;
2030             }
2031           b0 ^= sw_if_index0;
2032
2033           hash_v3_finalize32 (a0, b0, c0);
2034
2035           c0 &= BITS (hash_bitmap) - 1;
2036           c0 = c0 / BITS (uword);
2037           m0 = (uword) 1 << (c0 % BITS (uword));
2038
2039           bm0 = hash_bitmap[c0];
2040           drop0 = (bm0 & m0) != 0;
2041
2042           /* Mark it as seen. */
2043           hash_bitmap[c0] = bm0 | m0;
2044
2045           from += 1;
2046           n_left_from -= 1;
2047           to_next_drop[0] = pi0;
2048           to_next_drop += 1;
2049           n_left_to_next_drop -= 1;
2050
2051           p0->error =
2052             node->errors[drop0 ? IP4_ARP_ERROR_DROP :
2053                          IP4_ARP_ERROR_REQUEST_SENT];
2054
2055           /*
2056            * the adj has been updated to a rewrite but the node the DPO that got
2057            * us here hasn't - yet. no big deal. we'll drop while we wait.
2058            */
2059           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
2060             continue;
2061
2062           if (drop0)
2063             continue;
2064
2065           /*
2066            * Can happen if the control-plane is programming tables
2067            * with traffic flowing; at least that's today's lame excuse.
2068            */
2069           if ((is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_GLEAN)
2070               || (!is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP))
2071             {
2072               p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
2073             }
2074           else
2075             /* Send ARP request. */
2076             {
2077               u32 bi0 = 0;
2078               vlib_buffer_t *b0;
2079               ethernet_arp_header_t *h0;
2080               vnet_hw_interface_t *hw_if0;
2081
2082               h0 =
2083                 vlib_packet_template_get_packet (vm,
2084                                                  &im->ip4_arp_request_packet_template,
2085                                                  &bi0);
2086
2087               /* Add rewrite/encap string for ARP packet. */
2088               vnet_rewrite_one_header (adj0[0], h0,
2089                                        sizeof (ethernet_header_t));
2090
2091               hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2092
2093               /* Src ethernet address in ARP header. */
2094               clib_memcpy (h0->ip4_over_ethernet[0].ethernet,
2095                            hw_if0->hw_address,
2096                            sizeof (h0->ip4_over_ethernet[0].ethernet));
2097
2098               if (is_glean)
2099                 {
2100                   /* The interface's source address is stashed in the Glean Adj */
2101                   h0->ip4_over_ethernet[0].ip4 =
2102                     adj0->sub_type.glean.receive_addr.ip4;
2103
2104                   /* Copy in destination address we are requesting. This is the
2105                    * glean case, so it's the packet's destination.*/
2106                   h0->ip4_over_ethernet[1].ip4.data_u32 =
2107                     ip0->dst_address.data_u32;
2108                 }
2109               else
2110                 {
2111                   /* Src IP address in ARP header. */
2112                   if (ip4_src_address_for_packet (lm, sw_if_index0,
2113                                                   &h0->
2114                                                   ip4_over_ethernet[0].ip4))
2115                     {
2116                       /* No source address available */
2117                       p0->error =
2118                         node->errors[IP4_ARP_ERROR_NO_SOURCE_ADDRESS];
2119                       vlib_buffer_free (vm, &bi0, 1);
2120                       continue;
2121                     }
2122
2123                   /* Copy in destination address we are requesting from the
2124                      incomplete adj */
2125                   h0->ip4_over_ethernet[1].ip4.data_u32 =
2126                     adj0->sub_type.nbr.next_hop.ip4.as_u32;
2127                 }
2128
2129               vlib_buffer_copy_trace_flag (vm, p0, bi0);
2130               b0 = vlib_get_buffer (vm, bi0);
2131               vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
2132
2133               vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2134
2135               vlib_set_next_frame_buffer (vm, node,
2136                                           adj0->rewrite_header.next_index,
2137                                           bi0);
2138             }
2139         }
2140
2141       vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
2142     }
2143
2144   return frame->n_vectors;
2145 }
2146
2147 static uword
2148 ip4_arp (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
2149 {
2150   return (ip4_arp_inline (vm, node, frame, 0));
2151 }
2152
2153 static uword
2154 ip4_glean (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
2155 {
2156   return (ip4_arp_inline (vm, node, frame, 1));
2157 }
2158
2159 static char *ip4_arp_error_strings[] = {
2160   [IP4_ARP_ERROR_DROP] = "address overflow drops",
2161   [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
2162   [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
2163   [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
2164   [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
2165   [IP4_ARP_ERROR_NO_SOURCE_ADDRESS] = "no source address for ARP request",
2166 };
2167
2168 VLIB_REGISTER_NODE (ip4_arp_node) =
2169 {
2170   .function = ip4_arp,.name = "ip4-arp",.vector_size =
2171     sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_errors =
2172     ARRAY_LEN (ip4_arp_error_strings),.error_strings =
2173     ip4_arp_error_strings,.n_next_nodes = IP4_ARP_N_NEXT,.next_nodes =
2174   {
2175   [IP4_ARP_NEXT_DROP] = "error-drop",}
2176 ,};
2177
2178 VLIB_REGISTER_NODE (ip4_glean_node) =
2179 {
2180   .function = ip4_glean,.name = "ip4-glean",.vector_size =
2181     sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_errors =
2182     ARRAY_LEN (ip4_arp_error_strings),.error_strings =
2183     ip4_arp_error_strings,.n_next_nodes = IP4_ARP_N_NEXT,.next_nodes =
2184   {
2185   [IP4_ARP_NEXT_DROP] = "error-drop",}
2186 ,};
2187
2188 #define foreach_notrace_ip4_arp_error           \
2189 _(DROP)                                         \
2190 _(REQUEST_SENT)                                 \
2191 _(REPLICATE_DROP)                               \
2192 _(REPLICATE_FAIL)
2193
2194 clib_error_t *
2195 arp_notrace_init (vlib_main_t * vm)
2196 {
2197   vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, ip4_arp_node.index);
2198
2199   /* don't trace ARP request packets */
2200 #define _(a)                                    \
2201     vnet_pcap_drop_trace_filter_add_del         \
2202         (rt->errors[IP4_ARP_ERROR_##a],         \
2203          1 /* is_add */);
2204   foreach_notrace_ip4_arp_error;
2205 #undef _
2206   return 0;
2207 }
2208
2209 VLIB_INIT_FUNCTION (arp_notrace_init);
2210
2211
2212 /* Send an ARP request to see if given destination is reachable on given interface. */
2213 clib_error_t *
2214 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
2215 {
2216   vnet_main_t *vnm = vnet_get_main ();
2217   ip4_main_t *im = &ip4_main;
2218   ethernet_arp_header_t *h;
2219   ip4_address_t *src;
2220   ip_interface_address_t *ia;
2221   ip_adjacency_t *adj;
2222   vnet_hw_interface_t *hi;
2223   vnet_sw_interface_t *si;
2224   vlib_buffer_t *b;
2225   adj_index_t ai;
2226   u32 bi = 0;
2227
2228   si = vnet_get_sw_interface (vnm, sw_if_index);
2229
2230   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2231     {
2232       return clib_error_return (0, "%U: interface %U down",
2233                                 format_ip4_address, dst,
2234                                 format_vnet_sw_if_index_name, vnm,
2235                                 sw_if_index);
2236     }
2237
2238   src =
2239     ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2240   if (!src)
2241     {
2242       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2243       return clib_error_return
2244         (0,
2245          "no matching interface address for destination %U (interface %U)",
2246          format_ip4_address, dst, format_vnet_sw_if_index_name, vnm,
2247          sw_if_index);
2248     }
2249
2250   ip46_address_t nh = {
2251     .ip4 = *dst,
2252   };
2253
2254   ai = adj_nbr_add_or_lock (FIB_PROTOCOL_IP4,
2255                             VNET_LINK_IP4, &nh, sw_if_index);
2256   adj = adj_get (ai);
2257
2258   h = vlib_packet_template_get_packet (vm,
2259                                        &im->ip4_arp_request_packet_template,
2260                                        &bi);
2261
2262   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2263   if (PREDICT_FALSE (!hi->hw_address))
2264     {
2265       return clib_error_return (0, "%U: interface %U do not support ip probe",
2266                                 format_ip4_address, dst,
2267                                 format_vnet_sw_if_index_name, vnm,
2268                                 sw_if_index);
2269     }
2270
2271   clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address,
2272                sizeof (h->ip4_over_ethernet[0].ethernet));
2273
2274   h->ip4_over_ethernet[0].ip4 = src[0];
2275   h->ip4_over_ethernet[1].ip4 = dst[0];
2276
2277   b = vlib_get_buffer (vm, bi);
2278   vnet_buffer (b)->sw_if_index[VLIB_RX] =
2279     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2280
2281   /* Add encapsulation string for software interface (e.g. ethernet header). */
2282   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2283   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2284
2285   {
2286     vlib_frame_t *f = vlib_get_frame_to_node (vm, hi->output_node_index);
2287     u32 *to_next = vlib_frame_vector_args (f);
2288     to_next[0] = bi;
2289     f->n_vectors = 1;
2290     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2291   }
2292
2293   adj_unlock (ai);
2294   return /* no error */ 0;
2295 }
2296
2297 typedef enum
2298 {
2299   IP4_REWRITE_NEXT_DROP,
2300   IP4_REWRITE_NEXT_ICMP_ERROR,
2301 } ip4_rewrite_next_t;
2302
2303 always_inline uword
2304 ip4_rewrite_inline (vlib_main_t * vm,
2305                     vlib_node_runtime_t * node,
2306                     vlib_frame_t * frame,
2307                     int do_counters, int is_midchain, int is_mcast)
2308 {
2309   ip_lookup_main_t *lm = &ip4_main.lookup_main;
2310   u32 *from = vlib_frame_vector_args (frame);
2311   u32 n_left_from, n_left_to_next, *to_next, next_index;
2312   vlib_node_runtime_t *error_node =
2313     vlib_node_get_runtime (vm, ip4_input_node.index);
2314
2315   n_left_from = frame->n_vectors;
2316   next_index = node->cached_next_index;
2317   u32 thread_index = vlib_get_thread_index ();
2318
2319   while (n_left_from > 0)
2320     {
2321       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2322
2323       while (n_left_from >= 4 && n_left_to_next >= 2)
2324         {
2325           ip_adjacency_t *adj0, *adj1;
2326           vlib_buffer_t *p0, *p1;
2327           ip4_header_t *ip0, *ip1;
2328           u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
2329           u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
2330           u32 tx_sw_if_index0, tx_sw_if_index1;
2331
2332           /* Prefetch next iteration. */
2333           {
2334             vlib_buffer_t *p2, *p3;
2335
2336             p2 = vlib_get_buffer (vm, from[2]);
2337             p3 = vlib_get_buffer (vm, from[3]);
2338
2339             vlib_prefetch_buffer_header (p2, STORE);
2340             vlib_prefetch_buffer_header (p3, STORE);
2341
2342             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2343             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2344           }
2345
2346           pi0 = to_next[0] = from[0];
2347           pi1 = to_next[1] = from[1];
2348
2349           from += 2;
2350           n_left_from -= 2;
2351           to_next += 2;
2352           n_left_to_next -= 2;
2353
2354           p0 = vlib_get_buffer (vm, pi0);
2355           p1 = vlib_get_buffer (vm, pi1);
2356
2357           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2358           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
2359
2360           /*
2361            * pre-fetch the per-adjacency counters
2362            */
2363           if (do_counters)
2364             {
2365               vlib_prefetch_combined_counter (&adjacency_counters,
2366                                               thread_index, adj_index0);
2367               vlib_prefetch_combined_counter (&adjacency_counters,
2368                                               thread_index, adj_index1);
2369             }
2370
2371           ip0 = vlib_buffer_get_current (p0);
2372           ip1 = vlib_buffer_get_current (p1);
2373
2374           error0 = error1 = IP4_ERROR_NONE;
2375           next0 = next1 = IP4_REWRITE_NEXT_DROP;
2376
2377           /* Decrement TTL & update checksum.
2378              Works either endian, so no need for byte swap. */
2379           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2380             {
2381               i32 ttl0 = ip0->ttl;
2382
2383               /* Input node should have reject packets with ttl 0. */
2384               ASSERT (ip0->ttl > 0);
2385
2386               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2387               checksum0 += checksum0 >= 0xffff;
2388
2389               ip0->checksum = checksum0;
2390               ttl0 -= 1;
2391               ip0->ttl = ttl0;
2392
2393               /*
2394                * If the ttl drops below 1 when forwarding, generate
2395                * an ICMP response.
2396                */
2397               if (PREDICT_FALSE (ttl0 <= 0))
2398                 {
2399                   error0 = IP4_ERROR_TIME_EXPIRED;
2400                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2401                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2402                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2403                                                0);
2404                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2405                 }
2406
2407               /* Verify checksum. */
2408               ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
2409                       (p0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2410             }
2411           else
2412             {
2413               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2414             }
2415           if (PREDICT_TRUE (!(p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2416             {
2417               i32 ttl1 = ip1->ttl;
2418
2419               /* Input node should have reject packets with ttl 0. */
2420               ASSERT (ip1->ttl > 0);
2421
2422               checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2423               checksum1 += checksum1 >= 0xffff;
2424
2425               ip1->checksum = checksum1;
2426               ttl1 -= 1;
2427               ip1->ttl = ttl1;
2428
2429               /*
2430                * If the ttl drops below 1 when forwarding, generate
2431                * an ICMP response.
2432                */
2433               if (PREDICT_FALSE (ttl1 <= 0))
2434                 {
2435                   error1 = IP4_ERROR_TIME_EXPIRED;
2436                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2437                   icmp4_error_set_vnet_buffer (p1, ICMP4_time_exceeded,
2438                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2439                                                0);
2440                   next1 = IP4_REWRITE_NEXT_ICMP_ERROR;
2441                 }
2442
2443               /* Verify checksum. */
2444               ASSERT ((ip1->checksum == ip4_header_checksum (ip1)) ||
2445                       (p1->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2446             }
2447           else
2448             {
2449               p1->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2450             }
2451
2452           /* Rewrite packet header and updates lengths. */
2453           adj0 = adj_get (adj_index0);
2454           adj1 = adj_get (adj_index1);
2455
2456           /* Worth pipelining. No guarantee that adj0,1 are hot... */
2457           rw_len0 = adj0[0].rewrite_header.data_bytes;
2458           rw_len1 = adj1[0].rewrite_header.data_bytes;
2459           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2460           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
2461
2462           /* Check MTU of outgoing interface. */
2463           error0 =
2464             (vlib_buffer_length_in_chain (vm, p0) >
2465              adj0[0].
2466              rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
2467              error0);
2468           error1 =
2469             (vlib_buffer_length_in_chain (vm, p1) >
2470              adj1[0].
2471              rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
2472              error1);
2473
2474           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2475            * to see the IP headerr */
2476           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2477             {
2478               next0 = adj0[0].rewrite_header.next_index;
2479               p0->current_data -= rw_len0;
2480               p0->current_length += rw_len0;
2481               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2482               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2483
2484               if (PREDICT_FALSE
2485                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2486                 vnet_feature_arc_start (lm->output_feature_arc_index,
2487                                         tx_sw_if_index0, &next0, p0);
2488             }
2489           if (PREDICT_TRUE (error1 == IP4_ERROR_NONE))
2490             {
2491               next1 = adj1[0].rewrite_header.next_index;
2492               p1->current_data -= rw_len1;
2493               p1->current_length += rw_len1;
2494
2495               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
2496               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
2497
2498               if (PREDICT_FALSE
2499                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2500                 vnet_feature_arc_start (lm->output_feature_arc_index,
2501                                         tx_sw_if_index1, &next1, p1);
2502             }
2503
2504           /* Guess we are only writing on simple Ethernet header. */
2505           vnet_rewrite_two_headers (adj0[0], adj1[0],
2506                                     ip0, ip1, sizeof (ethernet_header_t));
2507
2508           /*
2509            * Bump the per-adjacency counters
2510            */
2511           if (do_counters)
2512             {
2513               vlib_increment_combined_counter
2514                 (&adjacency_counters,
2515                  thread_index,
2516                  adj_index0, 1,
2517                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2518
2519               vlib_increment_combined_counter
2520                 (&adjacency_counters,
2521                  thread_index,
2522                  adj_index1, 1,
2523                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
2524             }
2525
2526           if (is_midchain)
2527             {
2528               adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
2529               adj1->sub_type.midchain.fixup_func (vm, adj1, p1);
2530             }
2531           if (is_mcast)
2532             {
2533               /*
2534                * copy bytes from the IP address into the MAC rewrite
2535                */
2536               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2537               vnet_fixup_one_header (adj1[0], &ip1->dst_address, ip1);
2538             }
2539
2540           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2541                                            to_next, n_left_to_next,
2542                                            pi0, pi1, next0, next1);
2543         }
2544
2545       while (n_left_from > 0 && n_left_to_next > 0)
2546         {
2547           ip_adjacency_t *adj0;
2548           vlib_buffer_t *p0;
2549           ip4_header_t *ip0;
2550           u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2551           u32 tx_sw_if_index0;
2552
2553           pi0 = to_next[0] = from[0];
2554
2555           p0 = vlib_get_buffer (vm, pi0);
2556
2557           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2558
2559           adj0 = adj_get (adj_index0);
2560
2561           ip0 = vlib_buffer_get_current (p0);
2562
2563           error0 = IP4_ERROR_NONE;
2564           next0 = IP4_REWRITE_NEXT_DROP;        /* drop on error */
2565
2566           /* Decrement TTL & update checksum. */
2567           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2568             {
2569               i32 ttl0 = ip0->ttl;
2570
2571               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2572
2573               checksum0 += checksum0 >= 0xffff;
2574
2575               ip0->checksum = checksum0;
2576
2577               ASSERT (ip0->ttl > 0);
2578
2579               ttl0 -= 1;
2580
2581               ip0->ttl = ttl0;
2582
2583               ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
2584                       (p0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2585
2586               if (PREDICT_FALSE (ttl0 <= 0))
2587                 {
2588                   /*
2589                    * If the ttl drops below 1 when forwarding, generate
2590                    * an ICMP response.
2591                    */
2592                   error0 = IP4_ERROR_TIME_EXPIRED;
2593                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2594                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2595                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2596                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2597                                                0);
2598                 }
2599             }
2600           else
2601             {
2602               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2603             }
2604
2605           if (do_counters)
2606             vlib_prefetch_combined_counter (&adjacency_counters,
2607                                             thread_index, adj_index0);
2608
2609           /* Guess we are only writing on simple Ethernet header. */
2610           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2611           if (is_mcast)
2612             {
2613               /*
2614                * copy bytes from the IP address into the MAC rewrite
2615                */
2616               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2617             }
2618
2619           /* Update packet buffer attributes/set output interface. */
2620           rw_len0 = adj0[0].rewrite_header.data_bytes;
2621           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2622
2623           if (do_counters)
2624             vlib_increment_combined_counter
2625               (&adjacency_counters,
2626                thread_index, adj_index0, 1,
2627                vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2628
2629           /* Check MTU of outgoing interface. */
2630           error0 = (vlib_buffer_length_in_chain (vm, p0)
2631                     > adj0[0].rewrite_header.max_l3_packet_bytes
2632                     ? IP4_ERROR_MTU_EXCEEDED : error0);
2633
2634           p0->error = error_node->errors[error0];
2635
2636           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2637            * to see the IP headerr */
2638           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2639             {
2640               p0->current_data -= rw_len0;
2641               p0->current_length += rw_len0;
2642               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2643
2644               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2645               next0 = adj0[0].rewrite_header.next_index;
2646
2647               if (is_midchain)
2648                 {
2649                   adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
2650                 }
2651
2652               if (PREDICT_FALSE
2653                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2654                 vnet_feature_arc_start (lm->output_feature_arc_index,
2655                                         tx_sw_if_index0, &next0, p0);
2656
2657             }
2658
2659           from += 1;
2660           n_left_from -= 1;
2661           to_next += 1;
2662           n_left_to_next -= 1;
2663
2664           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2665                                            to_next, n_left_to_next,
2666                                            pi0, next0);
2667         }
2668
2669       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2670     }
2671
2672   /* Need to do trace after rewrites to pick up new packet data. */
2673   if (node->flags & VLIB_NODE_FLAG_TRACE)
2674     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2675
2676   return frame->n_vectors;
2677 }
2678
2679
2680 /** @brief IPv4 rewrite node.
2681     @node ip4-rewrite
2682
2683     This is the IPv4 transit-rewrite node: decrement TTL, fix the ipv4
2684     header checksum, fetch the ip adjacency, check the outbound mtu,
2685     apply the adjacency rewrite, and send pkts to the adjacency
2686     rewrite header's rewrite_next_index.
2687
2688     @param vm vlib_main_t corresponding to the current thread
2689     @param node vlib_node_runtime_t
2690     @param frame vlib_frame_t whose contents should be dispatched
2691
2692     @par Graph mechanics: buffer metadata, next index usage
2693
2694     @em Uses:
2695     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
2696         - the rewrite adjacency index
2697     - <code>adj->lookup_next_index</code>
2698         - Must be IP_LOOKUP_NEXT_REWRITE or IP_LOOKUP_NEXT_ARP, otherwise
2699           the packet will be dropped.
2700     - <code>adj->rewrite_header</code>
2701         - Rewrite string length, rewrite string, next_index
2702
2703     @em Sets:
2704     - <code>b->current_data, b->current_length</code>
2705         - Updated net of applying the rewrite string
2706
2707     <em>Next Indices:</em>
2708     - <code> adj->rewrite_header.next_index </code>
2709       or @c error-drop
2710 */
2711 static uword
2712 ip4_rewrite (vlib_main_t * vm,
2713              vlib_node_runtime_t * node, vlib_frame_t * frame)
2714 {
2715   if (adj_are_counters_enabled ())
2716     return ip4_rewrite_inline (vm, node, frame, 1, 0, 0);
2717   else
2718     return ip4_rewrite_inline (vm, node, frame, 0, 0, 0);
2719 }
2720
2721 static uword
2722 ip4_midchain (vlib_main_t * vm,
2723               vlib_node_runtime_t * node, vlib_frame_t * frame)
2724 {
2725   if (adj_are_counters_enabled ())
2726     return ip4_rewrite_inline (vm, node, frame, 1, 1, 0);
2727   else
2728     return ip4_rewrite_inline (vm, node, frame, 0, 1, 0);
2729 }
2730
2731 static uword
2732 ip4_rewrite_mcast (vlib_main_t * vm,
2733                    vlib_node_runtime_t * node, vlib_frame_t * frame)
2734 {
2735   if (adj_are_counters_enabled ())
2736     return ip4_rewrite_inline (vm, node, frame, 1, 0, 1);
2737   else
2738     return ip4_rewrite_inline (vm, node, frame, 0, 0, 1);
2739 }
2740
2741 static uword
2742 ip4_mcast_midchain (vlib_main_t * vm,
2743                     vlib_node_runtime_t * node, vlib_frame_t * frame)
2744 {
2745   if (adj_are_counters_enabled ())
2746     return ip4_rewrite_inline (vm, node, frame, 1, 1, 1);
2747   else
2748     return ip4_rewrite_inline (vm, node, frame, 0, 1, 1);
2749 }
2750
2751 /* *INDENT-OFF* */
2752 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2753   .function = ip4_rewrite,
2754   .name = "ip4-rewrite",
2755   .vector_size = sizeof (u32),
2756
2757   .format_trace = format_ip4_rewrite_trace,
2758
2759   .n_next_nodes = 2,
2760   .next_nodes = {
2761     [IP4_REWRITE_NEXT_DROP] = "error-drop",
2762     [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2763   },
2764 };
2765 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite)
2766
2767 VLIB_REGISTER_NODE (ip4_rewrite_mcast_node) = {
2768   .function = ip4_rewrite_mcast,
2769   .name = "ip4-rewrite-mcast",
2770   .vector_size = sizeof (u32),
2771
2772   .format_trace = format_ip4_rewrite_trace,
2773   .sibling_of = "ip4-rewrite",
2774 };
2775 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_mcast_node, ip4_rewrite_mcast)
2776
2777 VLIB_REGISTER_NODE (ip4_mcast_midchain_node, static) = {
2778   .function = ip4_mcast_midchain,
2779   .name = "ip4-mcast-midchain",
2780   .vector_size = sizeof (u32),
2781
2782   .format_trace = format_ip4_rewrite_trace,
2783   .sibling_of = "ip4-rewrite",
2784 };
2785 VLIB_NODE_FUNCTION_MULTIARCH (ip4_mcast_midchain_node, ip4_mcast_midchain)
2786
2787 VLIB_REGISTER_NODE (ip4_midchain_node) = {
2788   .function = ip4_midchain,
2789   .name = "ip4-midchain",
2790   .vector_size = sizeof (u32),
2791   .format_trace = format_ip4_forward_next_trace,
2792   .sibling_of =  "ip4-rewrite",
2793 };
2794 VLIB_NODE_FUNCTION_MULTIARCH (ip4_midchain_node, ip4_midchain);
2795 /* *INDENT-ON */
2796
2797 static clib_error_t *
2798 add_del_interface_table (vlib_main_t * vm,
2799                          unformat_input_t * input, vlib_cli_command_t * cmd)
2800 {
2801   vnet_main_t *vnm = vnet_get_main ();
2802   ip_interface_address_t *ia;
2803   clib_error_t *error = 0;
2804   u32 sw_if_index, table_id;
2805
2806   sw_if_index = ~0;
2807
2808   if (!unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2809     {
2810       error = clib_error_return (0, "unknown interface `%U'",
2811                                  format_unformat_error, input);
2812       goto done;
2813     }
2814
2815   if (unformat (input, "%d", &table_id))
2816     ;
2817   else
2818     {
2819       error = clib_error_return (0, "expected table id `%U'",
2820                                  format_unformat_error, input);
2821       goto done;
2822     }
2823
2824   /*
2825    * If the interface already has in IP address, then a change int
2826    * VRF is not allowed. The IP address applied must first be removed.
2827    * We do not do that automatically here, since VPP has no knowledge
2828    * of whether thoses subnets are valid in the destination VRF.
2829    */
2830   /* *INDENT-OFF* */
2831   foreach_ip_interface_address (&ip4_main.lookup_main,
2832                                 ia, sw_if_index,
2833                                 1 /* honor unnumbered */,
2834   ({
2835       ip4_address_t * a;
2836
2837       a = ip_interface_address_get_address (&ip4_main.lookup_main, ia);
2838       error = clib_error_return (0, "interface %U has address %U",
2839                                  format_vnet_sw_if_index_name, vnm,
2840                                  sw_if_index,
2841                                  format_ip4_address, a);
2842       goto done;
2843    }));
2844    /* *INDENT-ON* */
2845
2846 {
2847   ip4_main_t *im = &ip4_main;
2848   u32 fib_index;
2849
2850   fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, table_id);
2851
2852   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
2853   im->fib_index_by_sw_if_index[sw_if_index] = fib_index;
2854
2855   fib_index = mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, table_id);
2856   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
2857   im->mfib_index_by_sw_if_index[sw_if_index] = fib_index;
2858 }
2859
2860 done:
2861 return error;
2862 }
2863
2864 /*?
2865  * Place the indicated interface into the supplied IPv4 FIB table (also known
2866  * as a VRF). If the FIB table does not exist, this command creates it. To
2867  * display the current IPv4 FIB table, use the command '<em>show ip fib</em>'.
2868  * FIB table will only be displayed if a route has been added to the table, or
2869  * an IP Address is assigned to an interface in the table (which adds a route
2870  * automatically).
2871  *
2872  * @note IP addresses added after setting the interface IP table are added to
2873  * the indicated FIB table. If an IP address is added prior to changing the
2874  * table then this is an error. The control plane must remove these addresses
2875  * first and then change the table. VPP will not automatically move the
2876  * addresses from the old to the new table as it does not know the validity
2877  * of such a change.
2878  *
2879  * @cliexpar
2880  * Example of how to add an interface to an IPv4 FIB table (where 2 is the table-id):
2881  * @cliexcmd{set interface ip table GigabitEthernet2/0/0 2}
2882  ?*/
2883 /* *INDENT-OFF* */
2884 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) =
2885 {
2886   .path = "set interface ip table",
2887   .function = add_del_interface_table,
2888   .short_help = "set interface ip table <interface> <table-id>",
2889 };
2890 /* *INDENT-ON* */
2891
2892 int
2893 ip4_lookup_validate (ip4_address_t * a, u32 fib_index0)
2894 {
2895   ip4_fib_mtrie_t *mtrie0;
2896   ip4_fib_mtrie_leaf_t leaf0;
2897   u32 lbi0;
2898
2899   mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
2900
2901   leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, a);
2902   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
2903   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
2904
2905   lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2906
2907   return lbi0 == ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0), a);
2908 }
2909
2910 static clib_error_t *
2911 test_lookup_command_fn (vlib_main_t * vm,
2912                         unformat_input_t * input, vlib_cli_command_t * cmd)
2913 {
2914   ip4_fib_t *fib;
2915   u32 table_id = 0;
2916   f64 count = 1;
2917   u32 n;
2918   int i;
2919   ip4_address_t ip4_base_address;
2920   u64 errors = 0;
2921
2922   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2923     {
2924       if (unformat (input, "table %d", &table_id))
2925         {
2926           /* Make sure the entry exists. */
2927           fib = ip4_fib_get (table_id);
2928           if ((fib) && (fib->index != table_id))
2929             return clib_error_return (0, "<fib-index> %d does not exist",
2930                                       table_id);
2931         }
2932       else if (unformat (input, "count %f", &count))
2933         ;
2934
2935       else if (unformat (input, "%U",
2936                          unformat_ip4_address, &ip4_base_address))
2937         ;
2938       else
2939         return clib_error_return (0, "unknown input `%U'",
2940                                   format_unformat_error, input);
2941     }
2942
2943   n = count;
2944
2945   for (i = 0; i < n; i++)
2946     {
2947       if (!ip4_lookup_validate (&ip4_base_address, table_id))
2948         errors++;
2949
2950       ip4_base_address.as_u32 =
2951         clib_host_to_net_u32 (1 +
2952                               clib_net_to_host_u32 (ip4_base_address.as_u32));
2953     }
2954
2955   if (errors)
2956     vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
2957   else
2958     vlib_cli_output (vm, "No errors in %d lookups\n", n);
2959
2960   return 0;
2961 }
2962
2963 /*?
2964  * Perform a lookup of an IPv4 Address (or range of addresses) in the
2965  * given FIB table to determine if there is a conflict with the
2966  * adjacency table. The fib-id can be determined by using the
2967  * '<em>show ip fib</em>' command. If fib-id is not entered, default value
2968  * of 0 is used.
2969  *
2970  * @todo This command uses fib-id, other commands use table-id (not
2971  * just a name, they are different indexes). Would like to change this
2972  * to table-id for consistency.
2973  *
2974  * @cliexpar
2975  * Example of how to run the test lookup command:
2976  * @cliexstart{test lookup 172.16.1.1 table 1 count 2}
2977  * No errors in 2 lookups
2978  * @cliexend
2979 ?*/
2980 /* *INDENT-OFF* */
2981 VLIB_CLI_COMMAND (lookup_test_command, static) =
2982 {
2983   .path = "test lookup",
2984   .short_help = "test lookup <ipv4-addr> [table <fib-id>] [count <nn>]",
2985   .function = test_lookup_command_fn,
2986 };
2987 /* *INDENT-ON* */
2988
2989 int
2990 vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
2991 {
2992   u32 fib_index;
2993
2994   fib_index = fib_table_find (FIB_PROTOCOL_IP4, table_id);
2995
2996   if (~0 == fib_index)
2997     return VNET_API_ERROR_NO_SUCH_FIB;
2998
2999   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP4,
3000                                   flow_hash_config);
3001
3002   return 0;
3003 }
3004
3005 static clib_error_t *
3006 set_ip_flow_hash_command_fn (vlib_main_t * vm,
3007                              unformat_input_t * input,
3008                              vlib_cli_command_t * cmd)
3009 {
3010   int matched = 0;
3011   u32 table_id = 0;
3012   u32 flow_hash_config = 0;
3013   int rv;
3014
3015   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3016     {
3017       if (unformat (input, "table %d", &table_id))
3018         matched = 1;
3019 #define _(a,v) \
3020     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
3021       foreach_flow_hash_bit
3022 #undef _
3023         else
3024         break;
3025     }
3026
3027   if (matched == 0)
3028     return clib_error_return (0, "unknown input `%U'",
3029                               format_unformat_error, input);
3030
3031   rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
3032   switch (rv)
3033     {
3034     case 0:
3035       break;
3036
3037     case VNET_API_ERROR_NO_SUCH_FIB:
3038       return clib_error_return (0, "no such FIB table %d", table_id);
3039
3040     default:
3041       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3042       break;
3043     }
3044
3045   return 0;
3046 }
3047
3048 /*?
3049  * Configure the set of IPv4 fields used by the flow hash.
3050  *
3051  * @cliexpar
3052  * Example of how to set the flow hash on a given table:
3053  * @cliexcmd{set ip flow-hash table 7 dst sport dport proto}
3054  * Example of display the configured flow hash:
3055  * @cliexstart{show ip fib}
3056  * ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
3057  * 0.0.0.0/0
3058  *   unicast-ip4-chain
3059  *   [@0]: dpo-load-balance: [index:0 buckets:1 uRPF:0 to:[0:0]]
3060  *     [0] [@0]: dpo-drop ip6
3061  * 0.0.0.0/32
3062  *   unicast-ip4-chain
3063  *   [@0]: dpo-load-balance: [index:1 buckets:1 uRPF:1 to:[0:0]]
3064  *     [0] [@0]: dpo-drop ip6
3065  * 224.0.0.0/8
3066  *   unicast-ip4-chain
3067  *   [@0]: dpo-load-balance: [index:3 buckets:1 uRPF:3 to:[0:0]]
3068  *     [0] [@0]: dpo-drop ip6
3069  * 6.0.1.2/32
3070  *   unicast-ip4-chain
3071  *   [@0]: dpo-load-balance: [index:30 buckets:1 uRPF:29 to:[0:0]]
3072  *     [0] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
3073  * 7.0.0.1/32
3074  *   unicast-ip4-chain
3075  *   [@0]: dpo-load-balance: [index:31 buckets:4 uRPF:30 to:[0:0]]
3076  *     [0] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3077  *     [1] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3078  *     [2] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3079  *     [3] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
3080  * 240.0.0.0/8
3081  *   unicast-ip4-chain
3082  *   [@0]: dpo-load-balance: [index:2 buckets:1 uRPF:2 to:[0:0]]
3083  *     [0] [@0]: dpo-drop ip6
3084  * 255.255.255.255/32
3085  *   unicast-ip4-chain
3086  *   [@0]: dpo-load-balance: [index:4 buckets:1 uRPF:4 to:[0:0]]
3087  *     [0] [@0]: dpo-drop ip6
3088  * ipv4-VRF:7, fib_index 1, flow hash: dst sport dport proto
3089  * 0.0.0.0/0
3090  *   unicast-ip4-chain
3091  *   [@0]: dpo-load-balance: [index:12 buckets:1 uRPF:11 to:[0:0]]
3092  *     [0] [@0]: dpo-drop ip6
3093  * 0.0.0.0/32
3094  *   unicast-ip4-chain
3095  *   [@0]: dpo-load-balance: [index:13 buckets:1 uRPF:12 to:[0:0]]
3096  *     [0] [@0]: dpo-drop ip6
3097  * 172.16.1.0/24
3098  *   unicast-ip4-chain
3099  *   [@0]: dpo-load-balance: [index:17 buckets:1 uRPF:16 to:[0:0]]
3100  *     [0] [@4]: ipv4-glean: af_packet0
3101  * 172.16.1.1/32
3102  *   unicast-ip4-chain
3103  *   [@0]: dpo-load-balance: [index:18 buckets:1 uRPF:17 to:[1:84]]
3104  *     [0] [@2]: dpo-receive: 172.16.1.1 on af_packet0
3105  * 172.16.1.2/32
3106  *   unicast-ip4-chain
3107  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
3108  *     [0] [@5]: ipv4 via 172.16.1.2 af_packet0: IP4: 02:fe:9e:70:7a:2b -> 26:a5:f6:9c:3a:36
3109  * 172.16.2.0/24
3110  *   unicast-ip4-chain
3111  *   [@0]: dpo-load-balance: [index:19 buckets:1 uRPF:18 to:[0:0]]
3112  *     [0] [@4]: ipv4-glean: af_packet1
3113  * 172.16.2.1/32
3114  *   unicast-ip4-chain
3115  *   [@0]: dpo-load-balance: [index:20 buckets:1 uRPF:19 to:[0:0]]
3116  *     [0] [@2]: dpo-receive: 172.16.2.1 on af_packet1
3117  * 224.0.0.0/8
3118  *   unicast-ip4-chain
3119  *   [@0]: dpo-load-balance: [index:15 buckets:1 uRPF:14 to:[0:0]]
3120  *     [0] [@0]: dpo-drop ip6
3121  * 240.0.0.0/8
3122  *   unicast-ip4-chain
3123  *   [@0]: dpo-load-balance: [index:14 buckets:1 uRPF:13 to:[0:0]]
3124  *     [0] [@0]: dpo-drop ip6
3125  * 255.255.255.255/32
3126  *   unicast-ip4-chain
3127  *   [@0]: dpo-load-balance: [index:16 buckets:1 uRPF:15 to:[0:0]]
3128  *     [0] [@0]: dpo-drop ip6
3129  * @cliexend
3130 ?*/
3131 /* *INDENT-OFF* */
3132 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) =
3133 {
3134   .path = "set ip flow-hash",
3135   .short_help =
3136   "set ip flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
3137   .function = set_ip_flow_hash_command_fn,
3138 };
3139 /* *INDENT-ON* */
3140
3141 int
3142 vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3143                              u32 table_index)
3144 {
3145   vnet_main_t *vnm = vnet_get_main ();
3146   vnet_interface_main_t *im = &vnm->interface_main;
3147   ip4_main_t *ipm = &ip4_main;
3148   ip_lookup_main_t *lm = &ipm->lookup_main;
3149   vnet_classify_main_t *cm = &vnet_classify_main;
3150   ip4_address_t *if_addr;
3151
3152   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3153     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3154
3155   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3156     return VNET_API_ERROR_NO_SUCH_ENTRY;
3157
3158   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3159   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
3160
3161   if_addr = ip4_interface_first_address (ipm, sw_if_index, NULL);
3162
3163   if (NULL != if_addr)
3164     {
3165       fib_prefix_t pfx = {
3166         .fp_len = 32,
3167         .fp_proto = FIB_PROTOCOL_IP4,
3168         .fp_addr.ip4 = *if_addr,
3169       };
3170       u32 fib_index;
3171
3172       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3173                                                        sw_if_index);
3174
3175
3176       if (table_index != (u32) ~ 0)
3177         {
3178           dpo_id_t dpo = DPO_INVALID;
3179
3180           dpo_set (&dpo,
3181                    DPO_CLASSIFY,
3182                    DPO_PROTO_IP4,
3183                    classify_dpo_create (DPO_PROTO_IP4, table_index));
3184
3185           fib_table_entry_special_dpo_add (fib_index,
3186                                            &pfx,
3187                                            FIB_SOURCE_CLASSIFY,
3188                                            FIB_ENTRY_FLAG_NONE, &dpo);
3189           dpo_reset (&dpo);
3190         }
3191       else
3192         {
3193           fib_table_entry_special_remove (fib_index,
3194                                           &pfx, FIB_SOURCE_CLASSIFY);
3195         }
3196     }
3197
3198   return 0;
3199 }
3200
3201 static clib_error_t *
3202 set_ip_classify_command_fn (vlib_main_t * vm,
3203                             unformat_input_t * input,
3204                             vlib_cli_command_t * cmd)
3205 {
3206   u32 table_index = ~0;
3207   int table_index_set = 0;
3208   u32 sw_if_index = ~0;
3209   int rv;
3210
3211   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3212     {
3213       if (unformat (input, "table-index %d", &table_index))
3214         table_index_set = 1;
3215       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3216                          vnet_get_main (), &sw_if_index))
3217         ;
3218       else
3219         break;
3220     }
3221
3222   if (table_index_set == 0)
3223     return clib_error_return (0, "classify table-index must be specified");
3224
3225   if (sw_if_index == ~0)
3226     return clib_error_return (0, "interface / subif must be specified");
3227
3228   rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
3229
3230   switch (rv)
3231     {
3232     case 0:
3233       break;
3234
3235     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3236       return clib_error_return (0, "No such interface");
3237
3238     case VNET_API_ERROR_NO_SUCH_ENTRY:
3239       return clib_error_return (0, "No such classifier table");
3240     }
3241   return 0;
3242 }
3243
3244 /*?
3245  * Assign a classification table to an interface. The classification
3246  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3247  * commands. Once the table is create, use this command to filter packets
3248  * on an interface.
3249  *
3250  * @cliexpar
3251  * Example of how to assign a classification table to an interface:
3252  * @cliexcmd{set ip classify intfc GigabitEthernet2/0/0 table-index 1}
3253 ?*/
3254 /* *INDENT-OFF* */
3255 VLIB_CLI_COMMAND (set_ip_classify_command, static) =
3256 {
3257     .path = "set ip classify",
3258     .short_help =
3259     "set ip classify intfc <interface> table-index <classify-idx>",
3260     .function = set_ip_classify_command_fn,
3261 };
3262 /* *INDENT-ON* */
3263
3264 /*
3265  * fd.io coding-style-patch-verification: ON
3266  *
3267  * Local Variables:
3268  * eval: (c-set-style "gnu")
3269  * End:
3270  */