src/vnet/ip/ip4_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip4_forward.c: IP v4 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  43 #include <vnet/ethernet/arp_packet.h>   /* for ethernet_arp_header_t */
  44 #include <vnet/ppp/ppp.h>
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vnet/api_errno.h>     /* for API error numbers */
  47 #include <vnet/fib/fib_table.h> /* for FIB table and entry creation */
  48 #include <vnet/fib/fib_entry.h> /* for FIB table and entry creation */
  49 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  50 #include <vnet/fib/ip4_fib.h>
  51 #include <vnet/dpo/load_balance.h>
  52 #include <vnet/dpo/load_balance_map.h>
  53 #include <vnet/dpo/classify_dpo.h>
  54 #include <vnet/mfib/mfib_table.h>       /* for mFIB table and entry creation */
  55
  56 /**
  57  * @file
  58  * @brief IPv4 Forwarding.
  59  *
  60  * This file contains the source code for IPv4 forwarding.
  61  */
  62
  63 void
  64 ip4_forward_next_trace (vlib_main_t * vm,
  65                         vlib_node_runtime_t * node,
  66                         vlib_frame_t * frame,
  67                         vlib_rx_or_tx_t which_adj_index);
  68
  69 always_inline uword
  70 ip4_lookup_inline (vlib_main_t * vm,
  71                    vlib_node_runtime_t * node,
  72                    vlib_frame_t * frame,
  73                    int lookup_for_responses_to_locally_received_packets)
  74 {
  75   ip4_main_t *im = &ip4_main;
  76   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters;
  77   u32 n_left_from, n_left_to_next, *from, *to_next;
  78   ip_lookup_next_t next;
  79   u32 thread_index = vlib_get_thread_index ();
  80
  81   from = vlib_frame_vector_args (frame);
  82   n_left_from = frame->n_vectors;
  83   next = node->cached_next_index;
  84
  85   while (n_left_from > 0)
  86     {
  87       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
  88
  89       while (n_left_from >= 8 && n_left_to_next >= 4)
  90         {
  91           vlib_buffer_t *p0, *p1, *p2, *p3;
  92           ip4_header_t *ip0, *ip1, *ip2, *ip3;
  93           ip_lookup_next_t next0, next1, next2, next3;
  94           const load_balance_t *lb0, *lb1, *lb2, *lb3;
  95           ip4_fib_mtrie_t *mtrie0, *mtrie1, *mtrie2, *mtrie3;
  96           ip4_fib_mtrie_leaf_t leaf0, leaf1, leaf2, leaf3;
  97           ip4_address_t *dst_addr0, *dst_addr1, *dst_addr2, *dst_addr3;
  98           u32 pi0, fib_index0, lb_index0;
  99           u32 pi1, fib_index1, lb_index1;
 100           u32 pi2, fib_index2, lb_index2;
 101           u32 pi3, fib_index3, lb_index3;
 102           flow_hash_config_t flow_hash_config0, flow_hash_config1;
 103           flow_hash_config_t flow_hash_config2, flow_hash_config3;
 104           u32 hash_c0, hash_c1, hash_c2, hash_c3;
 105           const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3;
 106
 107           /* Prefetch next iteration. */
 108           {
 109             vlib_buffer_t *p4, *p5, *p6, *p7;
 110
 111             p4 = vlib_get_buffer (vm, from[4]);
 112             p5 = vlib_get_buffer (vm, from[5]);
 113             p6 = vlib_get_buffer (vm, from[6]);
 114             p7 = vlib_get_buffer (vm, from[7]);
 115
 116             vlib_prefetch_buffer_header (p4, LOAD);
 117             vlib_prefetch_buffer_header (p5, LOAD);
 118             vlib_prefetch_buffer_header (p6, LOAD);
 119             vlib_prefetch_buffer_header (p7, LOAD);
 120
 121             CLIB_PREFETCH (p4->data, sizeof (ip0[0]), LOAD);
 122             CLIB_PREFETCH (p5->data, sizeof (ip0[0]), LOAD);
 123             CLIB_PREFETCH (p6->data, sizeof (ip0[0]), LOAD);
 124             CLIB_PREFETCH (p7->data, sizeof (ip0[0]), LOAD);
 125           }
 126
 127           pi0 = to_next[0] = from[0];
 128           pi1 = to_next[1] = from[1];
 129           pi2 = to_next[2] = from[2];
 130           pi3 = to_next[3] = from[3];
 131
 132           from += 4;
 133           to_next += 4;
 134           n_left_to_next -= 4;
 135           n_left_from -= 4;
 136
 137           p0 = vlib_get_buffer (vm, pi0);
 138           p1 = vlib_get_buffer (vm, pi1);
 139           p2 = vlib_get_buffer (vm, pi2);
 140           p3 = vlib_get_buffer (vm, pi3);
 141
 142           ip0 = vlib_buffer_get_current (p0);
 143           ip1 = vlib_buffer_get_current (p1);
 144           ip2 = vlib_buffer_get_current (p2);
 145           ip3 = vlib_buffer_get_current (p3);
 146
 147           dst_addr0 = &ip0->dst_address;
 148           dst_addr1 = &ip1->dst_address;
 149           dst_addr2 = &ip2->dst_address;
 150           dst_addr3 = &ip3->dst_address;
 151
 152           fib_index0 =
 153             vec_elt (im->fib_index_by_sw_if_index,
 154                      vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 155           fib_index1 =
 156             vec_elt (im->fib_index_by_sw_if_index,
 157                      vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 158           fib_index2 =
 159             vec_elt (im->fib_index_by_sw_if_index,
 160                      vnet_buffer (p2)->sw_if_index[VLIB_RX]);
 161           fib_index3 =
 162             vec_elt (im->fib_index_by_sw_if_index,
 163                      vnet_buffer (p3)->sw_if_index[VLIB_RX]);
 164           fib_index0 =
 165             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
 166              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 167           fib_index1 =
 168             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
 169              (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
 170           fib_index2 =
 171             (vnet_buffer (p2)->sw_if_index[VLIB_TX] ==
 172              (u32) ~ 0) ? fib_index2 : vnet_buffer (p2)->sw_if_index[VLIB_TX];
 173           fib_index3 =
 174             (vnet_buffer (p3)->sw_if_index[VLIB_TX] ==
 175              (u32) ~ 0) ? fib_index3 : vnet_buffer (p3)->sw_if_index[VLIB_TX];
 176
 177
 178           if (!lookup_for_responses_to_locally_received_packets)
 179             {
 180               mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 181               mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
 182               mtrie2 = &ip4_fib_get (fib_index2)->mtrie;
 183               mtrie3 = &ip4_fib_get (fib_index3)->mtrie;
 184
 185               leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
 186               leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, dst_addr1);
 187               leaf2 = ip4_fib_mtrie_lookup_step_one (mtrie2, dst_addr2);
 188               leaf3 = ip4_fib_mtrie_lookup_step_one (mtrie3, dst_addr3);
 189             }
 190
 191           if (!lookup_for_responses_to_locally_received_packets)
 192             {
 193               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 194               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
 195               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 2);
 196               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 2);
 197             }
 198
 199           if (!lookup_for_responses_to_locally_received_packets)
 200             {
 201               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 202               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
 203               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 3);
 204               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 3);
 205             }
 206
 207           if (lookup_for_responses_to_locally_received_packets)
 208             {
 209               lb_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 210               lb_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
 211               lb_index2 = vnet_buffer (p2)->ip.adj_index[VLIB_RX];
 212               lb_index3 = vnet_buffer (p3)->ip.adj_index[VLIB_RX];
 213             }
 214           else
 215             {
 216               lb_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 217               lb_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 218               lb_index2 = ip4_fib_mtrie_leaf_get_adj_index (leaf2);
 219               lb_index3 = ip4_fib_mtrie_leaf_get_adj_index (leaf3);
 220             }
 221
 222           ASSERT (lb_index0 && lb_index1 && lb_index2 && lb_index3);
 223           lb0 = load_balance_get (lb_index0);
 224           lb1 = load_balance_get (lb_index1);
 225           lb2 = load_balance_get (lb_index2);
 226           lb3 = load_balance_get (lb_index3);
 227
 228           ASSERT (lb0->lb_n_buckets > 0);
 229           ASSERT (is_pow2 (lb0->lb_n_buckets));
 230           ASSERT (lb1->lb_n_buckets > 0);
 231           ASSERT (is_pow2 (lb1->lb_n_buckets));
 232           ASSERT (lb2->lb_n_buckets > 0);
 233           ASSERT (is_pow2 (lb2->lb_n_buckets));
 234           ASSERT (lb3->lb_n_buckets > 0);
 235           ASSERT (is_pow2 (lb3->lb_n_buckets));
 236
 237           /* Use flow hash to compute multipath adjacency. */
 238           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 239           hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
 240           hash_c2 = vnet_buffer (p2)->ip.flow_hash = 0;
 241           hash_c3 = vnet_buffer (p3)->ip.flow_hash = 0;
 242           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 243             {
 244               flow_hash_config0 = lb0->lb_hash_config;
 245               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 246                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 247               dpo0 =
 248                 load_balance_get_fwd_bucket (lb0,
 249                                              (hash_c0 &
 250                                               (lb0->lb_n_buckets_minus_1)));
 251             }
 252           else
 253             {
 254               dpo0 = load_balance_get_bucket_i (lb0, 0);
 255             }
 256           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 257             {
 258               flow_hash_config1 = lb1->lb_hash_config;
 259               hash_c1 = vnet_buffer (p1)->ip.flow_hash =
 260                 ip4_compute_flow_hash (ip1, flow_hash_config1);
 261               dpo1 =
 262                 load_balance_get_fwd_bucket (lb1,
 263                                              (hash_c1 &
 264                                               (lb1->lb_n_buckets_minus_1)));
 265             }
 266           else
 267             {
 268               dpo1 = load_balance_get_bucket_i (lb1, 0);
 269             }
 270           if (PREDICT_FALSE (lb2->lb_n_buckets > 1))
 271             {
 272               flow_hash_config2 = lb2->lb_hash_config;
 273               hash_c2 = vnet_buffer (p2)->ip.flow_hash =
 274                 ip4_compute_flow_hash (ip2, flow_hash_config2);
 275               dpo2 =
 276                 load_balance_get_fwd_bucket (lb2,
 277                                              (hash_c2 &
 278                                               (lb2->lb_n_buckets_minus_1)));
 279             }
 280           else
 281             {
 282               dpo2 = load_balance_get_bucket_i (lb2, 0);
 283             }
 284           if (PREDICT_FALSE (lb3->lb_n_buckets > 1))
 285             {
 286               flow_hash_config3 = lb3->lb_hash_config;
 287               hash_c3 = vnet_buffer (p3)->ip.flow_hash =
 288                 ip4_compute_flow_hash (ip3, flow_hash_config3);
 289               dpo3 =
 290                 load_balance_get_fwd_bucket (lb3,
 291                                              (hash_c3 &
 292                                               (lb3->lb_n_buckets_minus_1)));
 293             }
 294           else
 295             {
 296               dpo3 = load_balance_get_bucket_i (lb3, 0);
 297             }
 298
 299           next0 = dpo0->dpoi_next_node;
 300           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 301           next1 = dpo1->dpoi_next_node;
 302           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 303           next2 = dpo2->dpoi_next_node;
 304           vnet_buffer (p2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
 305           next3 = dpo3->dpoi_next_node;
 306           vnet_buffer (p3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
 307
 308           vlib_increment_combined_counter
 309             (cm, thread_index, lb_index0, 1,
 310              vlib_buffer_length_in_chain (vm, p0));
 311           vlib_increment_combined_counter
 312             (cm, thread_index, lb_index1, 1,
 313              vlib_buffer_length_in_chain (vm, p1));
 314           vlib_increment_combined_counter
 315             (cm, thread_index, lb_index2, 1,
 316              vlib_buffer_length_in_chain (vm, p2));
 317           vlib_increment_combined_counter
 318             (cm, thread_index, lb_index3, 1,
 319              vlib_buffer_length_in_chain (vm, p3));
 320
 321           vlib_validate_buffer_enqueue_x4 (vm, node, next,
 322                                            to_next, n_left_to_next,
 323                                            pi0, pi1, pi2, pi3,
 324                                            next0, next1, next2, next3);
 325         }
 326
 327       while (n_left_from > 0 && n_left_to_next > 0)
 328         {
 329           vlib_buffer_t *p0;
 330           ip4_header_t *ip0;
 331           ip_lookup_next_t next0;
 332           const load_balance_t *lb0;
 333           ip4_fib_mtrie_t *mtrie0;
 334           ip4_fib_mtrie_leaf_t leaf0;
 335           ip4_address_t *dst_addr0;
 336           u32 pi0, fib_index0, lbi0;
 337           flow_hash_config_t flow_hash_config0;
 338           const dpo_id_t *dpo0;
 339           u32 hash_c0;
 340
 341           pi0 = from[0];
 342           to_next[0] = pi0;
 343
 344           p0 = vlib_get_buffer (vm, pi0);
 345
 346           ip0 = vlib_buffer_get_current (p0);
 347
 348           dst_addr0 = &ip0->dst_address;
 349
 350           fib_index0 =
 351             vec_elt (im->fib_index_by_sw_if_index,
 352                      vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 353           fib_index0 =
 354             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
 355              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 356
 357           if (!lookup_for_responses_to_locally_received_packets)
 358             {
 359               mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 360
 361               leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
 362             }
 363
 364           if (!lookup_for_responses_to_locally_received_packets)
 365             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 366
 367           if (!lookup_for_responses_to_locally_received_packets)
 368             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 369
 370           if (lookup_for_responses_to_locally_received_packets)
 371             lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 372           else
 373             {
 374               /* Handle default route. */
 375               lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 376             }
 377
 378           ASSERT (lbi0);
 379           lb0 = load_balance_get (lbi0);
 380
 381           ASSERT (lb0->lb_n_buckets > 0);
 382           ASSERT (is_pow2 (lb0->lb_n_buckets));
 383
 384           /* Use flow hash to compute multipath adjacency. */
 385           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 386           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 387             {
 388               flow_hash_config0 = lb0->lb_hash_config;
 389
 390               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 391                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 392               dpo0 =
 393                 load_balance_get_fwd_bucket (lb0,
 394                                              (hash_c0 &
 395                                               (lb0->lb_n_buckets_minus_1)));
 396             }
 397           else
 398             {
 399               dpo0 = load_balance_get_bucket_i (lb0, 0);
 400             }
 401
 402           next0 = dpo0->dpoi_next_node;
 403           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 404
 405           vlib_increment_combined_counter (cm, thread_index, lbi0, 1,
 406                                            vlib_buffer_length_in_chain (vm,
 407                                                                         p0));
 408
 409           from += 1;
 410           to_next += 1;
 411           n_left_to_next -= 1;
 412           n_left_from -= 1;
 413
 414           if (PREDICT_FALSE (next0 != next))
 415             {
 416               n_left_to_next += 1;
 417               vlib_put_next_frame (vm, node, next, n_left_to_next);
 418               next = next0;
 419               vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 420               to_next[0] = pi0;
 421               to_next += 1;
 422               n_left_to_next -= 1;
 423             }
 424         }
 425
 426       vlib_put_next_frame (vm, node, next, n_left_to_next);
 427     }
 428
 429   if (node->flags & VLIB_NODE_FLAG_TRACE)
 430     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 431
 432   return frame->n_vectors;
 433 }
 434
 435 /** @brief IPv4 lookup node.
 436     @node ip4-lookup
 437
 438     This is the main IPv4 lookup dispatch node.
 439
 440     @param vm vlib_main_t corresponding to the current thread
 441     @param node vlib_node_runtime_t
 442     @param frame vlib_frame_t whose contents should be dispatched
 443
 444     @par Graph mechanics: buffer metadata, next index usage
 445
 446     @em Uses:
 447     - <code>vnet_buffer(b)->sw_if_index[VLIB_RX]</code>
 448         - Indicates the @c sw_if_index value of the interface that the
 449           packet was received on.
 450     - <code>vnet_buffer(b)->sw_if_index[VLIB_TX]</code>
 451         - When the value is @c ~0 then the node performs a longest prefix
 452           match (LPM) for the packet destination address in the FIB attached
 453           to the receive interface.
 454         - Otherwise perform LPM for the packet destination address in the
 455           indicated FIB. In this case <code>[VLIB_TX]</code> is a FIB index
 456           value (0, 1, ...) and not a VRF id.
 457
 458     @em Sets:
 459     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
 460         - The lookup result adjacency index.
 461
 462     <em>Next Index:</em>
 463     - Dispatches the packet to the node index found in
 464       ip_adjacency_t @c adj->lookup_next_index
 465       (where @c adj is the lookup result adjacency).
 466 */
 467 static uword
 468 ip4_lookup (vlib_main_t * vm,
 469             vlib_node_runtime_t * node, vlib_frame_t * frame)
 470 {
 471   return ip4_lookup_inline (vm, node, frame,
 472                             /* lookup_for_responses_to_locally_received_packets */
 473                             0);
 474
 475 }
 476
 477 static u8 *format_ip4_lookup_trace (u8 * s, va_list * args);
 478
 479 VLIB_REGISTER_NODE (ip4_lookup_node) =
 480 {
 481 .function = ip4_lookup,.name = "ip4-lookup",.vector_size =
 482     sizeof (u32),.format_trace = format_ip4_lookup_trace,.n_next_nodes =
 483     IP_LOOKUP_N_NEXT,.next_nodes = IP4_LOOKUP_NEXT_NODES,};
 484
 485 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup);
 486
 487 always_inline uword
 488 ip4_load_balance (vlib_main_t * vm,
 489                   vlib_node_runtime_t * node, vlib_frame_t * frame)
 490 {
 491   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 492   u32 n_left_from, n_left_to_next, *from, *to_next;
 493   ip_lookup_next_t next;
 494   u32 thread_index = vlib_get_thread_index ();
 495
 496   from = vlib_frame_vector_args (frame);
 497   n_left_from = frame->n_vectors;
 498   next = node->cached_next_index;
 499
 500   if (node->flags & VLIB_NODE_FLAG_TRACE)
 501     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 502
 503   while (n_left_from > 0)
 504     {
 505       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 506
 507
 508       while (n_left_from >= 4 && n_left_to_next >= 2)
 509         {
 510           ip_lookup_next_t next0, next1;
 511           const load_balance_t *lb0, *lb1;
 512           vlib_buffer_t *p0, *p1;
 513           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 514           const ip4_header_t *ip0, *ip1;
 515           const dpo_id_t *dpo0, *dpo1;
 516
 517           /* Prefetch next iteration. */
 518           {
 519             vlib_buffer_t *p2, *p3;
 520
 521             p2 = vlib_get_buffer (vm, from[2]);
 522             p3 = vlib_get_buffer (vm, from[3]);
 523
 524             vlib_prefetch_buffer_header (p2, STORE);
 525             vlib_prefetch_buffer_header (p3, STORE);
 526
 527             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 528             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 529           }
 530
 531           pi0 = to_next[0] = from[0];
 532           pi1 = to_next[1] = from[1];
 533
 534           from += 2;
 535           n_left_from -= 2;
 536           to_next += 2;
 537           n_left_to_next -= 2;
 538
 539           p0 = vlib_get_buffer (vm, pi0);
 540           p1 = vlib_get_buffer (vm, pi1);
 541
 542           ip0 = vlib_buffer_get_current (p0);
 543           ip1 = vlib_buffer_get_current (p1);
 544           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 545           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 546
 547           lb0 = load_balance_get (lbi0);
 548           lb1 = load_balance_get (lbi1);
 549
 550           /*
 551            * this node is for via FIBs we can re-use the hash value from the
 552            * to node if present.
 553            * We don't want to use the same hash value at each level in the recursion
 554            * graph as that would lead to polarisation
 555            */
 556           hc0 = hc1 = 0;
 557
 558           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 559             {
 560               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 561                 {
 562                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 563                     vnet_buffer (p0)->ip.flow_hash >> 1;
 564                 }
 565               else
 566                 {
 567                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 568                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 569                 }
 570               dpo0 = load_balance_get_fwd_bucket
 571                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 572             }
 573           else
 574             {
 575               dpo0 = load_balance_get_bucket_i (lb0, 0);
 576             }
 577           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 578             {
 579               if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash))
 580                 {
 581                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 582                     vnet_buffer (p1)->ip.flow_hash >> 1;
 583                 }
 584               else
 585                 {
 586                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 587                     ip4_compute_flow_hash (ip1, lb1->lb_hash_config);
 588                 }
 589               dpo1 = load_balance_get_fwd_bucket
 590                 (lb1, (hc1 & (lb1->lb_n_buckets_minus_1)));
 591             }
 592           else
 593             {
 594               dpo1 = load_balance_get_bucket_i (lb1, 0);
 595             }
 596
 597           next0 = dpo0->dpoi_next_node;
 598           next1 = dpo1->dpoi_next_node;
 599
 600           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 601           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 602
 603           vlib_increment_combined_counter
 604             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 605           vlib_increment_combined_counter
 606             (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 607
 608           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 609                                            to_next, n_left_to_next,
 610                                            pi0, pi1, next0, next1);
 611         }
 612
 613       while (n_left_from > 0 && n_left_to_next > 0)
 614         {
 615           ip_lookup_next_t next0;
 616           const load_balance_t *lb0;
 617           vlib_buffer_t *p0;
 618           u32 pi0, lbi0, hc0;
 619           const ip4_header_t *ip0;
 620           const dpo_id_t *dpo0;
 621
 622           pi0 = from[0];
 623           to_next[0] = pi0;
 624           from += 1;
 625           to_next += 1;
 626           n_left_to_next -= 1;
 627           n_left_from -= 1;
 628
 629           p0 = vlib_get_buffer (vm, pi0);
 630
 631           ip0 = vlib_buffer_get_current (p0);
 632           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 633
 634           lb0 = load_balance_get (lbi0);
 635
 636           hc0 = 0;
 637           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 638             {
 639               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 640                 {
 641                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 642                     vnet_buffer (p0)->ip.flow_hash >> 1;
 643                 }
 644               else
 645                 {
 646                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 647                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 648                 }
 649               dpo0 = load_balance_get_fwd_bucket
 650                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 651             }
 652           else
 653             {
 654               dpo0 = load_balance_get_bucket_i (lb0, 0);
 655             }
 656
 657           next0 = dpo0->dpoi_next_node;
 658           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 659
 660           vlib_increment_combined_counter
 661             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 662
 663           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 664                                            to_next, n_left_to_next,
 665                                            pi0, next0);
 666         }
 667
 668       vlib_put_next_frame (vm, node, next, n_left_to_next);
 669     }
 670
 671   return frame->n_vectors;
 672 }
 673
 674 VLIB_REGISTER_NODE (ip4_load_balance_node) =
 675 {
 676 .function = ip4_load_balance,.name = "ip4-load-balance",.vector_size =
 677     sizeof (u32),.sibling_of = "ip4-lookup",.format_trace =
 678     format_ip4_lookup_trace,};
 679
 680 VLIB_NODE_FUNCTION_MULTIARCH (ip4_load_balance_node, ip4_load_balance);
 681
 682 /* get first interface address */
 683 ip4_address_t *
 684 ip4_interface_first_address (ip4_main_t * im, u32 sw_if_index,
 685                              ip_interface_address_t ** result_ia)
 686 {
 687   ip_lookup_main_t *lm = &im->lookup_main;
 688   ip_interface_address_t *ia = 0;
 689   ip4_address_t *result = 0;
 690
 691   /* *INDENT-OFF* */
 692   foreach_ip_interface_address
 693     (lm, ia, sw_if_index,
 694      1 /* honor unnumbered */ ,
 695      ({
 696        ip4_address_t * a =
 697          ip_interface_address_get_address (lm, ia);
 698        result = a;
 699        break;
 700      }));
 701   /* *INDENT-OFF* */
 702   if (result_ia)
 703     *result_ia = result ? ia : 0;
 704   return result;
 705 }
 706
 707 static void
 708 ip4_add_interface_routes (u32 sw_if_index,
 709                           ip4_main_t * im, u32 fib_index,
 710                           ip_interface_address_t * a)
 711 {
 712   ip_lookup_main_t *lm = &im->lookup_main;
 713   ip4_address_t *address = ip_interface_address_get_address (lm, a);
 714   fib_prefix_t pfx = {
 715     .fp_len = a->address_length,
 716     .fp_proto = FIB_PROTOCOL_IP4,
 717     .fp_addr.ip4 = *address,
 718   };
 719
 720   if (pfx.fp_len <= 30)
 721     {
 722       /* a /30 or shorter - add a glean for the network address */
 723       fib_table_entry_update_one_path (fib_index, &pfx,
 724                                        FIB_SOURCE_INTERFACE,
 725                                        (FIB_ENTRY_FLAG_CONNECTED |
 726                                         FIB_ENTRY_FLAG_ATTACHED),
 727                                        DPO_PROTO_IP4,
 728                                        /* No next-hop address */
 729                                        NULL,
 730                                        sw_if_index,
 731                                        // invalid FIB index
 732                                        ~0,
 733                                        1,
 734                                        // no out-label stack
 735                                        NULL,
 736                                        FIB_ROUTE_PATH_FLAG_NONE);
 737
 738       /* Add the two broadcast addresses as drop */
 739       fib_prefix_t net_pfx = {
 740         .fp_len = 32,
 741         .fp_proto = FIB_PROTOCOL_IP4,
 742         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 743       };
 744       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 745         fib_table_entry_special_add(fib_index,
 746                                     &net_pfx,
 747                                     FIB_SOURCE_INTERFACE,
 748                                     (FIB_ENTRY_FLAG_DROP |
 749                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 750       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 751       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 752         fib_table_entry_special_add(fib_index,
 753                                     &net_pfx,
 754                                     FIB_SOURCE_INTERFACE,
 755                                     (FIB_ENTRY_FLAG_DROP |
 756                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 757     }
 758   else if (pfx.fp_len == 31)
 759     {
 760       u32 mask = clib_host_to_net_u32(1);
 761       fib_prefix_t net_pfx = pfx;
 762
 763       net_pfx.fp_len = 32;
 764       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 765
 766       /* a /31 - add the other end as an attached host */
 767       fib_table_entry_update_one_path (fib_index, &net_pfx,
 768                                        FIB_SOURCE_INTERFACE,
 769                                        (FIB_ENTRY_FLAG_ATTACHED),
 770                                        DPO_PROTO_IP4,
 771                                        &net_pfx.fp_addr,
 772                                        sw_if_index,
 773                                        // invalid FIB index
 774                                        ~0,
 775                                        1,
 776                                        NULL,
 777                                        FIB_ROUTE_PATH_FLAG_NONE);
 778     }
 779   pfx.fp_len = 32;
 780
 781   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 782     {
 783       u32 classify_table_index =
 784         lm->classify_table_index_by_sw_if_index[sw_if_index];
 785       if (classify_table_index != (u32) ~ 0)
 786         {
 787           dpo_id_t dpo = DPO_INVALID;
 788
 789           dpo_set (&dpo,
 790                    DPO_CLASSIFY,
 791                    DPO_PROTO_IP4,
 792                    classify_dpo_create (DPO_PROTO_IP4, classify_table_index));
 793
 794           fib_table_entry_special_dpo_add (fib_index,
 795                                            &pfx,
 796                                            FIB_SOURCE_CLASSIFY,
 797                                            FIB_ENTRY_FLAG_NONE, &dpo);
 798           dpo_reset (&dpo);
 799         }
 800     }
 801
 802   fib_table_entry_update_one_path (fib_index, &pfx,
 803                                    FIB_SOURCE_INTERFACE,
 804                                    (FIB_ENTRY_FLAG_CONNECTED |
 805                                     FIB_ENTRY_FLAG_LOCAL),
 806                                    DPO_PROTO_IP4,
 807                                    &pfx.fp_addr,
 808                                    sw_if_index,
 809                                    // invalid FIB index
 810                                    ~0,
 811                                    1, NULL,
 812                                    FIB_ROUTE_PATH_FLAG_NONE);
 813 }
 814
 815 static void
 816 ip4_del_interface_routes (ip4_main_t * im,
 817                           u32 fib_index,
 818                           ip4_address_t * address, u32 address_length)
 819 {
 820   fib_prefix_t pfx = {
 821     .fp_len = address_length,
 822     .fp_proto = FIB_PROTOCOL_IP4,
 823     .fp_addr.ip4 = *address,
 824   };
 825
 826   if (pfx.fp_len <= 30)
 827     {
 828       fib_prefix_t net_pfx = {
 829         .fp_len = 32,
 830         .fp_proto = FIB_PROTOCOL_IP4,
 831         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 832       };
 833       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 834         fib_table_entry_special_remove(fib_index,
 835                                        &net_pfx,
 836                                        FIB_SOURCE_INTERFACE);
 837       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 838       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 839         fib_table_entry_special_remove(fib_index,
 840                                        &net_pfx,
 841                                        FIB_SOURCE_INTERFACE);
 842       fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 843     }
 844     else if (pfx.fp_len == 31)
 845     {
 846       u32 mask = clib_host_to_net_u32(1);
 847       fib_prefix_t net_pfx = pfx;
 848
 849       net_pfx.fp_len = 32;
 850       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 851
 852       fib_table_entry_delete (fib_index, &net_pfx, FIB_SOURCE_INTERFACE);
 853     }
 854
 855   pfx.fp_len = 32;
 856   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 857 }
 858
 859 void
 860 ip4_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 861 {
 862   ip4_main_t *im = &ip4_main;
 863
 864   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 865
 866   /*
 867    * enable/disable only on the 1<->0 transition
 868    */
 869   if (is_enable)
 870     {
 871       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 872         return;
 873     }
 874   else
 875     {
 876       ASSERT (im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 877       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 878         return;
 879     }
 880   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
 881                                !is_enable, 0, 0);
 882
 883
 884   vnet_feature_enable_disable ("ip4-multicast", "ip4-drop",
 885                                sw_if_index, !is_enable, 0, 0);
 886 }
 887
 888 static clib_error_t *
 889 ip4_add_del_interface_address_internal (vlib_main_t * vm,
 890                                         u32 sw_if_index,
 891                                         ip4_address_t * address,
 892                                         u32 address_length, u32 is_del)
 893 {
 894   vnet_main_t *vnm = vnet_get_main ();
 895   ip4_main_t *im = &ip4_main;
 896   ip_lookup_main_t *lm = &im->lookup_main;
 897   clib_error_t *error = 0;
 898   u32 if_address_index, elts_before;
 899   ip4_address_fib_t ip4_af, *addr_fib = 0;
 900
 901   /* local0 interface doesn't support IP addressing  */
 902   if (sw_if_index == 0)
 903     {
 904       return
 905        clib_error_create ("local0 interface doesn't support IP addressing");
 906     }
 907
 908   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 909   ip4_addr_fib_init (&ip4_af, address,
 910                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 911   vec_add1 (addr_fib, ip4_af);
 912
 913   /* FIXME-LATER
 914    * there is no support for adj-fib handling in the presence of overlapping
 915    * subnets on interfaces. Easy fix - disallow overlapping subnets, like
 916    * most routers do.
 917    */
 918   /* *INDENT-OFF* */
 919   if (!is_del)
 920     {
 921       /* When adding an address check that it does not conflict
 922          with an existing address. */
 923       ip_interface_address_t *ia;
 924       foreach_ip_interface_address
 925         (&im->lookup_main, ia, sw_if_index,
 926          0 /* honor unnumbered */ ,
 927          ({
 928            ip4_address_t * x =
 929              ip_interface_address_get_address
 930              (&im->lookup_main, ia);
 931            if (ip4_destination_matches_route
 932                (im, address, x, ia->address_length) ||
 933                ip4_destination_matches_route (im,
 934                                               x,
 935                                               address,
 936                                               address_length))
 937              return
 938                clib_error_create
 939                ("failed to add %U which conflicts with %U for interface %U",
 940                 format_ip4_address_and_length, address,
 941                 address_length,
 942                 format_ip4_address_and_length, x,
 943                 ia->address_length,
 944                 format_vnet_sw_if_index_name, vnm,
 945                 sw_if_index);
 946          }));
 947     }
 948   /* *INDENT-ON* */
 949
 950   elts_before = pool_elts (lm->if_address_pool);
 951
 952   error = ip_interface_address_add_del
 953     (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 954   if (error)
 955     goto done;
 956
 957   ip4_sw_interface_enable_disable (sw_if_index, !is_del);
 958
 959   if (is_del)
 960     ip4_del_interface_routes (im, ip4_af.fib_index, address, address_length);
 961   else
 962     ip4_add_interface_routes (sw_if_index,
 963                               im, ip4_af.fib_index,
 964                               pool_elt_at_index
 965                               (lm->if_address_pool, if_address_index));
 966
 967   /* If pool did not grow/shrink: add duplicate address. */
 968   if (elts_before != pool_elts (lm->if_address_pool))
 969     {
 970       ip4_add_del_interface_address_callback_t *cb;
 971       vec_foreach (cb, im->add_del_interface_address_callbacks)
 972         cb->function (im, cb->function_opaque, sw_if_index,
 973                       address, address_length, if_address_index, is_del);
 974     }
 975
 976 done:
 977   vec_free (addr_fib);
 978   return error;
 979 }
 980
 981 clib_error_t *
 982 ip4_add_del_interface_address (vlib_main_t * vm,
 983                                u32 sw_if_index,
 984                                ip4_address_t * address,
 985                                u32 address_length, u32 is_del)
 986 {
 987   return ip4_add_del_interface_address_internal
 988     (vm, sw_if_index, address, address_length, is_del);
 989 }
 990
 991 /* Built-in ip4 unicast rx feature path definition */
 992 /* *INDENT-OFF* */
 993 VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
 994 {
 995   .arc_name = "ip4-unicast",
 996   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 997   .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
 998 };
 999
1000 VNET_FEATURE_INIT (ip4_flow_classify, static) =
1001 {
1002   .arc_name = "ip4-unicast",
1003   .node_name = "ip4-flow-classify",
1004   .runs_before = VNET_FEATURES ("ip4-inacl"),
1005 };
1006
1007 VNET_FEATURE_INIT (ip4_inacl, static) =
1008 {
1009   .arc_name = "ip4-unicast",
1010   .node_name = "ip4-inacl",
1011   .runs_before = VNET_FEATURES ("ip4-source-check-via-rx"),
1012 };
1013
1014 VNET_FEATURE_INIT (ip4_source_check_1, static) =
1015 {
1016   .arc_name = "ip4-unicast",
1017   .node_name = "ip4-source-check-via-rx",
1018   .runs_before = VNET_FEATURES ("ip4-source-check-via-any"),
1019 };
1020
1021 VNET_FEATURE_INIT (ip4_source_check_2, static) =
1022 {
1023   .arc_name = "ip4-unicast",
1024   .node_name = "ip4-source-check-via-any",
1025   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
1026 };
1027
1028 VNET_FEATURE_INIT (ip4_source_and_port_range_check_rx, static) =
1029 {
1030   .arc_name = "ip4-unicast",
1031   .node_name = "ip4-source-and-port-range-check-rx",
1032   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
1033 };
1034
1035 VNET_FEATURE_INIT (ip4_policer_classify, static) =
1036 {
1037   .arc_name = "ip4-unicast",
1038   .node_name = "ip4-policer-classify",
1039   .runs_before = VNET_FEATURES ("ipsec-input-ip4"),
1040 };
1041
1042 VNET_FEATURE_INIT (ip4_ipsec, static) =
1043 {
1044   .arc_name = "ip4-unicast",
1045   .node_name = "ipsec-input-ip4",
1046   .runs_before = VNET_FEATURES ("vpath-input-ip4"),
1047 };
1048
1049 VNET_FEATURE_INIT (ip4_vpath, static) =
1050 {
1051   .arc_name = "ip4-unicast",
1052   .node_name = "vpath-input-ip4",
1053   .runs_before = VNET_FEATURES ("ip4-vxlan-bypass"),
1054 };
1055
1056 VNET_FEATURE_INIT (ip4_vxlan_bypass, static) =
1057 {
1058   .arc_name = "ip4-unicast",
1059   .node_name = "ip4-vxlan-bypass",
1060   .runs_before = VNET_FEATURES ("ip4-lookup"),
1061 };
1062
1063 VNET_FEATURE_INIT (ip4_drop, static) =
1064 {
1065   .arc_name = "ip4-unicast",
1066   .node_name = "ip4-drop",
1067   .runs_before = VNET_FEATURES ("ip4-lookup"),
1068 };
1069
1070 VNET_FEATURE_INIT (ip4_lookup, static) =
1071 {
1072   .arc_name = "ip4-unicast",
1073   .node_name = "ip4-lookup",
1074   .runs_before = 0,     /* not before any other features */
1075 };
1076
1077 /* Built-in ip4 multicast rx feature path definition */
1078 VNET_FEATURE_ARC_INIT (ip4_multicast, static) =
1079 {
1080   .arc_name = "ip4-multicast",
1081   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
1082   .arc_index_ptr = &ip4_main.lookup_main.mcast_feature_arc_index,
1083 };
1084
1085 VNET_FEATURE_INIT (ip4_vpath_mc, static) =
1086 {
1087   .arc_name = "ip4-multicast",
1088   .node_name = "vpath-input-ip4",
1089   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
1090 };
1091
1092 VNET_FEATURE_INIT (ip4_mc_drop, static) =
1093 {
1094   .arc_name = "ip4-multicast",
1095   .node_name = "ip4-drop",
1096   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
1097 };
1098
1099 VNET_FEATURE_INIT (ip4_lookup_mc, static) =
1100 {
1101   .arc_name = "ip4-multicast",
1102   .node_name = "ip4-mfib-forward-lookup",
1103   .runs_before = 0,     /* last feature */
1104 };
1105
1106 /* Source and port-range check ip4 tx feature path definition */
1107 VNET_FEATURE_ARC_INIT (ip4_output, static) =
1108 {
1109   .arc_name = "ip4-output",
1110   .start_nodes = VNET_FEATURES ("ip4-rewrite", "ip4-midchain"),
1111   .arc_index_ptr = &ip4_main.lookup_main.output_feature_arc_index,
1112 };
1113
1114 VNET_FEATURE_INIT (ip4_source_and_port_range_check_tx, static) =
1115 {
1116   .arc_name = "ip4-output",
1117   .node_name = "ip4-source-and-port-range-check-tx",
1118   .runs_before = VNET_FEATURES ("ipsec-output-ip4"),
1119 };
1120
1121 VNET_FEATURE_INIT (ip4_ipsec_output, static) =
1122 {
1123   .arc_name = "ip4-output",
1124   .node_name = "ipsec-output-ip4",
1125   .runs_before = VNET_FEATURES ("interface-output"),
1126 };
1127
1128 /* Built-in ip4 tx feature path definition */
1129 VNET_FEATURE_INIT (ip4_interface_output, static) =
1130 {
1131   .arc_name = "ip4-output",
1132   .node_name = "interface-output",
1133   .runs_before = 0,     /* not before any other features */
1134 };
1135 /* *INDENT-ON* */
1136
1137 static clib_error_t *
1138 ip4_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
1139 {
1140   ip4_main_t *im = &ip4_main;
1141
1142   /* Fill in lookup tables with default table (0). */
1143   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1144   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
1145
1146   if (!is_add)
1147     {
1148       ip4_main_t *im4 = &ip4_main;
1149       ip_lookup_main_t *lm4 = &im4->lookup_main;
1150       ip_interface_address_t *ia = 0;
1151       ip4_address_t *address;
1152       vlib_main_t *vm = vlib_get_main ();
1153
1154       /* *INDENT-OFF* */
1155       foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* honor unnumbered */,
1156       ({
1157         address = ip_interface_address_get_address (lm4, ia);
1158         ip4_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
1159       }));
1160       /* *INDENT-ON* */
1161     }
1162
1163   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
1164                                is_add, 0, 0);
1165
1166   vnet_feature_enable_disable ("ip4-multicast", "ip4-drop", sw_if_index,
1167                                is_add, 0, 0);
1168
1169   return /* no error */ 0;
1170 }
1171
1172 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
1173
1174 /* Global IP4 main. */
1175 ip4_main_t ip4_main;
1176
1177 clib_error_t *
1178 ip4_lookup_init (vlib_main_t * vm)
1179 {
1180   ip4_main_t *im = &ip4_main;
1181   clib_error_t *error;
1182   uword i;
1183
1184   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
1185     return error;
1186
1187   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
1188     {
1189       u32 m;
1190
1191       if (i < 32)
1192         m = pow2_mask (i) << (32 - i);
1193       else
1194         m = ~0;
1195       im->fib_masks[i] = clib_host_to_net_u32 (m);
1196     }
1197
1198   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
1199
1200   /* Create FIB with index 0 and table id of 0. */
1201   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0,
1202                                      FIB_SOURCE_DEFAULT_ROUTE);
1203   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0,
1204                                       MFIB_SOURCE_DEFAULT_ROUTE);
1205
1206   {
1207     pg_node_t *pn;
1208     pn = pg_get_node (ip4_lookup_node.index);
1209     pn->unformat_edit = unformat_pg_ip4_header;
1210   }
1211
1212   {
1213     ethernet_arp_header_t h;
1214
1215     memset (&h, 0, sizeof (h));
1216
1217     /* Set target ethernet address to all zeros. */
1218     memset (h.ip4_over_ethernet[1].ethernet, 0,
1219             sizeof (h.ip4_over_ethernet[1].ethernet));
1220
1221 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
1222 #define _8(f,v) h.f = v;
1223     _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
1224     _16 (l3_type, ETHERNET_TYPE_IP4);
1225     _8 (n_l2_address_bytes, 6);
1226     _8 (n_l3_address_bytes, 4);
1227     _16 (opcode, ETHERNET_ARP_OPCODE_request);
1228 #undef _16
1229 #undef _8
1230
1231     vlib_packet_template_init (vm, &im->ip4_arp_request_packet_template,
1232                                /* data */ &h,
1233                                sizeof (h),
1234                                /* alloc chunk size */ 8,
1235                                "ip4 arp");
1236   }
1237
1238   return error;
1239 }
1240
1241 VLIB_INIT_FUNCTION (ip4_lookup_init);
1242
1243 typedef struct
1244 {
1245   /* Adjacency taken. */
1246   u32 dpo_index;
1247   u32 flow_hash;
1248   u32 fib_index;
1249
1250   /* Packet data, possibly *after* rewrite. */
1251   u8 packet_data[64 - 1 * sizeof (u32)];
1252 }
1253 ip4_forward_next_trace_t;
1254
1255 u8 *
1256 format_ip4_forward_next_trace (u8 * s, va_list * args)
1257 {
1258   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1259   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1260   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1261   u32 indent = format_get_indent (s);
1262   s = format (s, "%U%U",
1263               format_white_space, indent,
1264               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1265   return s;
1266 }
1267
1268 static u8 *
1269 format_ip4_lookup_trace (u8 * s, va_list * args)
1270 {
1271   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1272   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1273   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1274   u32 indent = format_get_indent (s);
1275
1276   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
1277               t->fib_index, t->dpo_index, t->flow_hash);
1278   s = format (s, "\n%U%U",
1279               format_white_space, indent,
1280               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1281   return s;
1282 }
1283
1284 static u8 *
1285 format_ip4_rewrite_trace (u8 * s, va_list * args)
1286 {
1287   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1288   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1289   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1290   u32 indent = format_get_indent (s);
1291
1292   s = format (s, "tx_sw_if_index %d dpo-idx %d : %U flow hash: 0x%08x",
1293               t->fib_index, t->dpo_index, format_ip_adjacency,
1294               t->dpo_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
1295   s = format (s, "\n%U%U",
1296               format_white_space, indent,
1297               format_ip_adjacency_packet_data,
1298               t->dpo_index, t->packet_data, sizeof (t->packet_data));
1299   return s;
1300 }
1301
1302 /* Common trace function for all ip4-forward next nodes. */
1303 void
1304 ip4_forward_next_trace (vlib_main_t * vm,
1305                         vlib_node_runtime_t * node,
1306                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
1307 {
1308   u32 *from, n_left;
1309   ip4_main_t *im = &ip4_main;
1310
1311   n_left = frame->n_vectors;
1312   from = vlib_frame_vector_args (frame);
1313
1314   while (n_left >= 4)
1315     {
1316       u32 bi0, bi1;
1317       vlib_buffer_t *b0, *b1;
1318       ip4_forward_next_trace_t *t0, *t1;
1319
1320       /* Prefetch next iteration. */
1321       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1322       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1323
1324       bi0 = from[0];
1325       bi1 = from[1];
1326
1327       b0 = vlib_get_buffer (vm, bi0);
1328       b1 = vlib_get_buffer (vm, bi1);
1329
1330       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1331         {
1332           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1333           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1334           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1335           t0->fib_index =
1336             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1337              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1338             vec_elt (im->fib_index_by_sw_if_index,
1339                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1340
1341           clib_memcpy (t0->packet_data,
1342                        vlib_buffer_get_current (b0),
1343                        sizeof (t0->packet_data));
1344         }
1345       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1346         {
1347           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1348           t1->dpo_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1349           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1350           t1->fib_index =
1351             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
1352              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
1353             vec_elt (im->fib_index_by_sw_if_index,
1354                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
1355           clib_memcpy (t1->packet_data, vlib_buffer_get_current (b1),
1356                        sizeof (t1->packet_data));
1357         }
1358       from += 2;
1359       n_left -= 2;
1360     }
1361
1362   while (n_left >= 1)
1363     {
1364       u32 bi0;
1365       vlib_buffer_t *b0;
1366       ip4_forward_next_trace_t *t0;
1367
1368       bi0 = from[0];
1369
1370       b0 = vlib_get_buffer (vm, bi0);
1371
1372       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1373         {
1374           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1375           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1376           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1377           t0->fib_index =
1378             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1379              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1380             vec_elt (im->fib_index_by_sw_if_index,
1381                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1382           clib_memcpy (t0->packet_data, vlib_buffer_get_current (b0),
1383                        sizeof (t0->packet_data));
1384         }
1385       from += 1;
1386       n_left -= 1;
1387     }
1388 }
1389
1390 static uword
1391 ip4_drop_or_punt (vlib_main_t * vm,
1392                   vlib_node_runtime_t * node,
1393                   vlib_frame_t * frame, ip4_error_t error_code)
1394 {
1395   u32 *buffers = vlib_frame_vector_args (frame);
1396   uword n_packets = frame->n_vectors;
1397
1398   vlib_error_drop_buffers (vm, node, buffers,
1399                            /* stride */ 1,
1400                            n_packets,
1401                            /* next */ 0,
1402                            ip4_input_node.index, error_code);
1403
1404   if (node->flags & VLIB_NODE_FLAG_TRACE)
1405     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1406
1407   return n_packets;
1408 }
1409
1410 static uword
1411 ip4_drop (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1412 {
1413   return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_DROP);
1414 }
1415
1416 static uword
1417 ip4_punt (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1418 {
1419   return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_PUNT);
1420 }
1421
1422 /* *INDENT-OFF* */
1423 VLIB_REGISTER_NODE (ip4_drop_node, static) =
1424 {
1425   .function = ip4_drop,
1426   .name = "ip4-drop",
1427   .vector_size = sizeof (u32),
1428   .format_trace = format_ip4_forward_next_trace,
1429   .n_next_nodes = 1,
1430   .next_nodes = {
1431     [0] = "error-drop",
1432   },
1433 };
1434
1435 VLIB_NODE_FUNCTION_MULTIARCH (ip4_drop_node, ip4_drop);
1436
1437 VLIB_REGISTER_NODE (ip4_punt_node, static) =
1438 {
1439   .function = ip4_punt,
1440   .name = "ip4-punt",
1441   .vector_size = sizeof (u32),
1442   .format_trace = format_ip4_forward_next_trace,
1443   .n_next_nodes = 1,
1444   .next_nodes = {
1445     [0] = "error-punt",
1446   },
1447 };
1448
1449 VLIB_NODE_FUNCTION_MULTIARCH (ip4_punt_node, ip4_punt);
1450 /* *INDENT-ON */
1451
1452 /* Compute TCP/UDP/ICMP4 checksum in software. */
1453 u16
1454 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1455                               ip4_header_t * ip0)
1456 {
1457   ip_csum_t sum0;
1458   u32 ip_header_length, payload_length_host_byte_order;
1459   u32 n_this_buffer, n_bytes_left, n_ip_bytes_this_buffer;
1460   u16 sum16;
1461   void *data_this_buffer;
1462
1463   /* Initialize checksum with ip header. */
1464   ip_header_length = ip4_header_bytes (ip0);
1465   payload_length_host_byte_order =
1466     clib_net_to_host_u16 (ip0->length) - ip_header_length;
1467   sum0 =
1468     clib_host_to_net_u32 (payload_length_host_byte_order +
1469                           (ip0->protocol << 16));
1470
1471   if (BITS (uword) == 32)
1472     {
1473       sum0 =
1474         ip_csum_with_carry (sum0,
1475                             clib_mem_unaligned (&ip0->src_address, u32));
1476       sum0 =
1477         ip_csum_with_carry (sum0,
1478                             clib_mem_unaligned (&ip0->dst_address, u32));
1479     }
1480   else
1481     sum0 =
1482       ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1483
1484   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1485   data_this_buffer = (void *) ip0 + ip_header_length;
1486   n_ip_bytes_this_buffer = p0->current_length - (((u8 *) ip0 - p0->data) - p0->current_data);
1487   if (n_this_buffer + ip_header_length > n_ip_bytes_this_buffer)
1488     {
1489       n_this_buffer = n_ip_bytes_this_buffer > ip_header_length ?
1490           n_ip_bytes_this_buffer - ip_header_length : 0;
1491     }
1492   while (1)
1493     {
1494       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1495       n_bytes_left -= n_this_buffer;
1496       if (n_bytes_left == 0)
1497         break;
1498
1499       ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1500       p0 = vlib_get_buffer (vm, p0->next_buffer);
1501       data_this_buffer = vlib_buffer_get_current (p0);
1502       n_this_buffer = p0->current_length;
1503     }
1504
1505   sum16 = ~ip_csum_fold (sum0);
1506
1507   return sum16;
1508 }
1509
1510 u32
1511 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1512 {
1513   ip4_header_t *ip0 = vlib_buffer_get_current (p0);
1514   udp_header_t *udp0;
1515   u16 sum16;
1516
1517   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1518           || ip0->protocol == IP_PROTOCOL_UDP);
1519
1520   udp0 = (void *) (ip0 + 1);
1521   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1522     {
1523       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1524                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
1525       return p0->flags;
1526     }
1527
1528   sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1529
1530   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1531                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
1532
1533   return p0->flags;
1534 }
1535
1536 /* *INDENT-OFF* */
1537 VNET_FEATURE_ARC_INIT (ip4_local) =
1538 {
1539   .arc_name  = "ip4-local",
1540   .start_nodes = VNET_FEATURES ("ip4-local"),
1541 };
1542 /* *INDENT-ON* */
1543
1544 static inline void
1545 ip4_local_validate_l4 (vlib_main_t * vm, vlib_buffer_t * p, ip4_header_t * ip,
1546                        u8 is_udp, u8 * error, u8 * good_tcp_udp)
1547 {
1548   u32 flags0;
1549   flags0 = ip4_tcp_udp_validate_checksum (vm, p);
1550   *good_tcp_udp = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1551   if (is_udp)
1552     {
1553       udp_header_t *udp;
1554       u32 ip_len, udp_len;
1555       i32 len_diff;
1556       udp = ip4_next_header (ip);
1557       /* Verify UDP length. */
1558       ip_len = clib_net_to_host_u16 (ip->length);
1559       udp_len = clib_net_to_host_u16 (udp->length);
1560
1561       len_diff = ip_len - udp_len;
1562       *good_tcp_udp &= len_diff >= 0;
1563       *error = len_diff < 0 ? IP4_ERROR_UDP_LENGTH : *error;
1564     }
1565 }
1566
1567 #define ip4_local_do_l4_check(is_tcp_udp, flags)                        \
1568     (is_tcp_udp && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED))
1569
1570 static inline uword
1571 ip4_local_inline (vlib_main_t * vm,
1572                   vlib_node_runtime_t * node,
1573                   vlib_frame_t * frame, int head_of_feature_arc)
1574 {
1575   ip4_main_t *im = &ip4_main;
1576   ip_lookup_main_t *lm = &im->lookup_main;
1577   ip_local_next_t next_index;
1578   u32 *from, *to_next, n_left_from, n_left_to_next;
1579   vlib_node_runtime_t *error_node =
1580     vlib_node_get_runtime (vm, ip4_input_node.index);
1581   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1582
1583   from = vlib_frame_vector_args (frame);
1584   n_left_from = frame->n_vectors;
1585   next_index = node->cached_next_index;
1586
1587   if (node->flags & VLIB_NODE_FLAG_TRACE)
1588     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1589
1590   while (n_left_from > 0)
1591     {
1592       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1593
1594       while (n_left_from >= 4 && n_left_to_next >= 2)
1595         {
1596           vlib_buffer_t *p0, *p1;
1597           ip4_header_t *ip0, *ip1;
1598           ip4_fib_mtrie_t *mtrie0, *mtrie1;
1599           ip4_fib_mtrie_leaf_t leaf0, leaf1;
1600           const dpo_id_t *dpo0, *dpo1;
1601           const load_balance_t *lb0, *lb1;
1602           u32 pi0, next0, fib_index0, lbi0;
1603           u32 pi1, next1, fib_index1, lbi1;
1604           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1605           u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1606           u32 sw_if_index0, sw_if_index1;
1607
1608           pi0 = to_next[0] = from[0];
1609           pi1 = to_next[1] = from[1];
1610           from += 2;
1611           n_left_from -= 2;
1612           to_next += 2;
1613           n_left_to_next -= 2;
1614
1615           next0 = next1 = IP_LOCAL_NEXT_DROP;
1616           error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1617
1618           p0 = vlib_get_buffer (vm, pi0);
1619           p1 = vlib_get_buffer (vm, pi1);
1620
1621           ip0 = vlib_buffer_get_current (p0);
1622           ip1 = vlib_buffer_get_current (p1);
1623
1624           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1625           vnet_buffer (p1)->l3_hdr_offset = p1->current_data;
1626
1627           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1628           sw_if_index1 = vnet_buffer (p1)->sw_if_index[VLIB_RX];
1629
1630           /* Treat IP frag packets as "experimental" protocol for now
1631              until support of IP frag reassembly is implemented */
1632           proto0 = ip4_is_fragment (ip0) ? 0xfe : ip0->protocol;
1633           proto1 = ip4_is_fragment (ip1) ? 0xfe : ip1->protocol;
1634
1635           if (head_of_feature_arc == 0)
1636             goto skip_checks;
1637
1638           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1639           is_udp1 = proto1 == IP_PROTOCOL_UDP;
1640           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1641           is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1642
1643           good_tcp_udp0 =
1644             (p0->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1645           good_tcp_udp1 =
1646             (p1->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1647
1648           if (PREDICT_FALSE (ip4_local_do_l4_check (is_tcp_udp0, p0->flags)
1649                              || ip4_local_do_l4_check (is_tcp_udp1,
1650                                                        p1->flags)))
1651             {
1652               if (is_tcp_udp0)
1653                 ip4_local_validate_l4 (vm, p0, ip0, is_udp0, &error0,
1654                                        &good_tcp_udp0);
1655               if (is_tcp_udp1)
1656                 ip4_local_validate_l4 (vm, p1, ip1, is_udp1, &error1,
1657                                        &good_tcp_udp1);
1658             }
1659
1660           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1661           error0 = (is_tcp_udp0 && !good_tcp_udp0
1662                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1663           error1 = (is_tcp_udp1 && !good_tcp_udp1
1664                     ? IP4_ERROR_TCP_CHECKSUM + is_udp1 : error1);
1665
1666           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1667           fib_index0 =
1668             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1669              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1670
1671           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index1);
1672           fib_index1 =
1673             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
1674              (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
1675
1676           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1677           mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
1678
1679           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1680           leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, &ip1->src_address);
1681           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1682                                              2);
1683           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address,
1684                                              2);
1685           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1686                                              3);
1687           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address,
1688                                              3);
1689
1690           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0 =
1691             ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1692           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1693
1694           vnet_buffer (p1)->ip.adj_index[VLIB_RX] = lbi1 =
1695             ip4_fib_mtrie_leaf_get_adj_index (leaf1);
1696           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = lbi1;
1697
1698           lb0 = load_balance_get (lbi0);
1699           lb1 = load_balance_get (lbi1);
1700           dpo0 = load_balance_get_bucket_i (lb0, 0);
1701           dpo1 = load_balance_get_bucket_i (lb1, 0);
1702
1703           /*
1704            * Must have a route to source otherwise we drop the packet.
1705            * ip4 broadcasts are accepted, e.g. to make dhcp client work
1706            *
1707            * The checks are:
1708            *  - the source is a recieve => it's from us => bogus, do this
1709            *    first since it sets a different error code.
1710            *  - uRPF check for any route to source - accept if passes.
1711            *  - allow packets destined to the broadcast address from unknown sources
1712            */
1713           if (p0->flags & VNET_BUFFER_F_IS_NATED)
1714             goto skip_check0;
1715
1716           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1717                      dpo0->dpoi_type == DPO_RECEIVE) ?
1718                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1719           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1720                      !fib_urpf_check_size (lb0->lb_urpf) &&
1721                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1722                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1723
1724         skip_check0:
1725           if (p1->flags & VNET_BUFFER_F_IS_NATED)
1726             goto skip_checks;
1727
1728           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1729                      dpo1->dpoi_type == DPO_RECEIVE) ?
1730                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error1);
1731           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1732                      !fib_urpf_check_size (lb1->lb_urpf) &&
1733                      ip1->dst_address.as_u32 != 0xFFFFFFFF)
1734                     ? IP4_ERROR_SRC_LOOKUP_MISS : error1);
1735
1736         skip_checks:
1737
1738           next0 = lm->local_next_by_ip_protocol[proto0];
1739           next1 = lm->local_next_by_ip_protocol[proto1];
1740
1741           next0 =
1742             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1743           next1 =
1744             error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1745
1746           p0->error = error0 ? error_node->errors[error0] : 0;
1747           p1->error = error1 ? error_node->errors[error1] : 0;
1748
1749           if (head_of_feature_arc)
1750             {
1751               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1752                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1753               if (PREDICT_TRUE (error1 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1754                 vnet_feature_arc_start (arc_index, sw_if_index1, &next1, p1);
1755             }
1756
1757           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1758                                            n_left_to_next, pi0, pi1,
1759                                            next0, next1);
1760         }
1761
1762       while (n_left_from > 0 && n_left_to_next > 0)
1763         {
1764           vlib_buffer_t *p0;
1765           ip4_header_t *ip0;
1766           ip4_fib_mtrie_t *mtrie0;
1767           ip4_fib_mtrie_leaf_t leaf0;
1768           u32 pi0, next0, fib_index0, lbi0;
1769           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1770           load_balance_t *lb0;
1771           const dpo_id_t *dpo0;
1772           u32 sw_if_index0;
1773
1774           pi0 = to_next[0] = from[0];
1775           from += 1;
1776           n_left_from -= 1;
1777           to_next += 1;
1778           n_left_to_next -= 1;
1779
1780           next0 = IP_LOCAL_NEXT_DROP;
1781           error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
1782
1783           p0 = vlib_get_buffer (vm, pi0);
1784           ip0 = vlib_buffer_get_current (p0);
1785           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1786           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1787
1788           /* Treat IP frag packets as "experimental" protocol for now
1789              until support of IP frag reassembly is implemented */
1790           proto0 = ip4_is_fragment (ip0) ? 0xfe : ip0->protocol;
1791
1792           if (head_of_feature_arc == 0 || p0->flags & VNET_BUFFER_F_IS_NATED)
1793             goto skip_check;
1794
1795           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1796           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1797           good_tcp_udp0 =
1798             (p0->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1799
1800           if (PREDICT_FALSE (ip4_local_do_l4_check (is_tcp_udp0, p0->flags)))
1801             {
1802               ip4_local_validate_l4 (vm, p0, ip0, is_udp0, &error0,
1803                                      &good_tcp_udp0);
1804             }
1805
1806           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1807           error0 = (is_tcp_udp0 && !good_tcp_udp0
1808                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1809
1810           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1811           fib_index0 =
1812             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1813              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1814           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1815           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1816           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1817                                              2);
1818           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1819                                              3);
1820           lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1821           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1822           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0;
1823
1824           lb0 = load_balance_get (lbi0);
1825           dpo0 = load_balance_get_bucket_i (lb0, 0);
1826
1827           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1828                      dpo0->dpoi_type == DPO_RECEIVE) ?
1829                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1830           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1831                      !fib_urpf_check_size (lb0->lb_urpf) &&
1832                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1833                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1834
1835         skip_check:
1836           next0 = lm->local_next_by_ip_protocol[proto0];
1837           next0 =
1838             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1839
1840           p0->error = error0 ? error_node->errors[error0] : 0;
1841
1842           if (head_of_feature_arc)
1843             {
1844               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1845                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1846             }
1847
1848           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1849                                            n_left_to_next, pi0, next0);
1850         }
1851       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1852     }
1853
1854   return frame->n_vectors;
1855 }
1856
1857 static uword
1858 ip4_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1859 {
1860   return ip4_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1861 }
1862
1863 /* *INDENT-OFF* */
1864 VLIB_REGISTER_NODE (ip4_local_node) =
1865 {
1866   .function = ip4_local,
1867   .name = "ip4-local",
1868   .vector_size = sizeof (u32),
1869   .format_trace = format_ip4_forward_next_trace,
1870   .n_next_nodes = IP_LOCAL_N_NEXT,
1871   .next_nodes =
1872   {
1873     [IP_LOCAL_NEXT_DROP] = "error-drop",
1874     [IP_LOCAL_NEXT_PUNT] = "error-punt",
1875     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
1876     [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",
1877   },
1878 };
1879 /* *INDENT-ON* */
1880
1881 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local);
1882
1883 static uword
1884 ip4_local_end_of_arc (vlib_main_t * vm,
1885                       vlib_node_runtime_t * node, vlib_frame_t * frame)
1886 {
1887   return ip4_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1888 }
1889
1890 /* *INDENT-OFF* */
1891 VLIB_REGISTER_NODE (ip4_local_end_of_arc_node,static) = {
1892   .function = ip4_local_end_of_arc,
1893   .name = "ip4-local-end-of-arc",
1894   .vector_size = sizeof (u32),
1895
1896   .format_trace = format_ip4_forward_next_trace,
1897   .sibling_of = "ip4-local",
1898 };
1899
1900 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_end_of_arc_node, ip4_local_end_of_arc)
1901
1902 VNET_FEATURE_INIT (ip4_local_end_of_arc, static) = {
1903   .arc_name = "ip4-local",
1904   .node_name = "ip4-local-end-of-arc",
1905   .runs_before = 0, /* not before any other features */
1906 };
1907 /* *INDENT-ON* */
1908
1909 void
1910 ip4_register_protocol (u32 protocol, u32 node_index)
1911 {
1912   vlib_main_t *vm = vlib_get_main ();
1913   ip4_main_t *im = &ip4_main;
1914   ip_lookup_main_t *lm = &im->lookup_main;
1915
1916   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1917   lm->local_next_by_ip_protocol[protocol] =
1918     vlib_node_add_next (vm, ip4_local_node.index, node_index);
1919 }
1920
1921 static clib_error_t *
1922 show_ip_local_command_fn (vlib_main_t * vm,
1923                           unformat_input_t * input, vlib_cli_command_t * cmd)
1924 {
1925   ip4_main_t *im = &ip4_main;
1926   ip_lookup_main_t *lm = &im->lookup_main;
1927   int i;
1928
1929   vlib_cli_output (vm, "Protocols handled by ip4_local");
1930   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
1931     {
1932       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
1933         {
1934           u32 node_index = vlib_get_node (vm,
1935                                           ip4_local_node.index)->
1936             next_nodes[lm->local_next_by_ip_protocol[i]];
1937           vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm,
1938                            node_index);
1939         }
1940     }
1941   return 0;
1942 }
1943
1944
1945
1946 /*?
1947  * Display the set of protocols handled by the local IPv4 stack.
1948  *
1949  * @cliexpar
1950  * Example of how to display local protocol table:
1951  * @cliexstart{show ip local}
1952  * Protocols handled by ip4_local
1953  * 1
1954  * 17
1955  * 47
1956  * @cliexend
1957 ?*/
1958 /* *INDENT-OFF* */
1959 VLIB_CLI_COMMAND (show_ip_local, static) =
1960 {
1961   .path = "show ip local",
1962   .function = show_ip_local_command_fn,
1963   .short_help = "show ip local",
1964 };
1965 /* *INDENT-ON* */
1966
1967 always_inline uword
1968 ip4_arp_inline (vlib_main_t * vm,
1969                 vlib_node_runtime_t * node,
1970                 vlib_frame_t * frame, int is_glean)
1971 {
1972   vnet_main_t *vnm = vnet_get_main ();
1973   ip4_main_t *im = &ip4_main;
1974   ip_lookup_main_t *lm = &im->lookup_main;
1975   u32 *from, *to_next_drop;
1976   uword n_left_from, n_left_to_next_drop, next_index;
1977   static f64 time_last_seed_change = -1e100;
1978   static u32 hash_seeds[3];
1979   static uword hash_bitmap[256 / BITS (uword)];
1980   f64 time_now;
1981
1982   if (node->flags & VLIB_NODE_FLAG_TRACE)
1983     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1984
1985   time_now = vlib_time_now (vm);
1986   if (time_now - time_last_seed_change > 1e-3)
1987     {
1988       uword i;
1989       u32 *r = clib_random_buffer_get_data (&vm->random_buffer,
1990                                             sizeof (hash_seeds));
1991       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1992         hash_seeds[i] = r[i];
1993
1994       /* Mark all hash keys as been no-seen before. */
1995       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1996         hash_bitmap[i] = 0;
1997
1998       time_last_seed_change = time_now;
1999     }
2000
2001   from = vlib_frame_vector_args (frame);
2002   n_left_from = frame->n_vectors;
2003   next_index = node->cached_next_index;
2004   if (next_index == IP4_ARP_NEXT_DROP)
2005     next_index = IP4_ARP_N_NEXT;        /* point to first interface */
2006
2007   while (n_left_from > 0)
2008     {
2009       vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
2010                            to_next_drop, n_left_to_next_drop);
2011
2012       while (n_left_from > 0 && n_left_to_next_drop > 0)
2013         {
2014           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
2015           ip_adjacency_t *adj0;
2016           vlib_buffer_t *p0;
2017           ip4_header_t *ip0;
2018           uword bm0;
2019
2020           pi0 = from[0];
2021
2022           p0 = vlib_get_buffer (vm, pi0);
2023
2024           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2025           adj0 = adj_get (adj_index0);
2026           ip0 = vlib_buffer_get_current (p0);
2027
2028           a0 = hash_seeds[0];
2029           b0 = hash_seeds[1];
2030           c0 = hash_seeds[2];
2031
2032           sw_if_index0 = adj0->rewrite_header.sw_if_index;
2033           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
2034
2035           if (is_glean)
2036             {
2037               /*
2038                * this is the Glean case, so we are ARPing for the
2039                * packet's destination
2040                */
2041               a0 ^= ip0->dst_address.data_u32;
2042             }
2043           else
2044             {
2045               a0 ^= adj0->sub_type.nbr.next_hop.ip4.data_u32;
2046             }
2047           b0 ^= sw_if_index0;
2048
2049           hash_v3_mix32 (a0, b0, c0);
2050           hash_v3_finalize32 (a0, b0, c0);
2051
2052           c0 &= BITS (hash_bitmap) - 1;
2053           m0 = (uword) 1 << (c0 % BITS (uword));
2054           c0 = c0 / BITS (uword);
2055
2056           bm0 = hash_bitmap[c0];
2057           drop0 = (bm0 & m0) != 0;
2058
2059           /* Mark it as seen. */
2060           hash_bitmap[c0] = bm0 | m0;
2061
2062           from += 1;
2063           n_left_from -= 1;
2064           to_next_drop[0] = pi0;
2065           to_next_drop += 1;
2066           n_left_to_next_drop -= 1;
2067
2068           p0->error =
2069             node->errors[drop0 ? IP4_ARP_ERROR_DROP :
2070                          IP4_ARP_ERROR_REQUEST_SENT];
2071
2072           /*
2073            * the adj has been updated to a rewrite but the node the DPO that got
2074            * us here hasn't - yet. no big deal. we'll drop while we wait.
2075            */
2076           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
2077             continue;
2078
2079           if (drop0)
2080             continue;
2081
2082           /*
2083            * Can happen if the control-plane is programming tables
2084            * with traffic flowing; at least that's today's lame excuse.
2085            */
2086           if ((is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_GLEAN)
2087               || (!is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP))
2088             {
2089               p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
2090             }
2091           else
2092             /* Send ARP request. */
2093             {
2094               u32 bi0 = 0;
2095               vlib_buffer_t *b0;
2096               ethernet_arp_header_t *h0;
2097               vnet_hw_interface_t *hw_if0;
2098
2099               h0 =
2100                 vlib_packet_template_get_packet (vm,
2101                                                  &im->ip4_arp_request_packet_template,
2102                                                  &bi0);
2103
2104               /* Add rewrite/encap string for ARP packet. */
2105               vnet_rewrite_one_header (adj0[0], h0,
2106                                        sizeof (ethernet_header_t));
2107
2108               hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2109
2110               /* Src ethernet address in ARP header. */
2111               clib_memcpy (h0->ip4_over_ethernet[0].ethernet,
2112                            hw_if0->hw_address,
2113                            sizeof (h0->ip4_over_ethernet[0].ethernet));
2114
2115               if (is_glean)
2116                 {
2117                   /* The interface's source address is stashed in the Glean Adj */
2118                   h0->ip4_over_ethernet[0].ip4 =
2119                     adj0->sub_type.glean.receive_addr.ip4;
2120
2121                   /* Copy in destination address we are requesting. This is the
2122                    * glean case, so it's the packet's destination.*/
2123                   h0->ip4_over_ethernet[1].ip4.data_u32 =
2124                     ip0->dst_address.data_u32;
2125                 }
2126               else
2127                 {
2128                   /* Src IP address in ARP header. */
2129                   if (ip4_src_address_for_packet (lm, sw_if_index0,
2130                                                   &h0->
2131                                                   ip4_over_ethernet[0].ip4))
2132                     {
2133                       /* No source address available */
2134                       p0->error =
2135                         node->errors[IP4_ARP_ERROR_NO_SOURCE_ADDRESS];
2136                       vlib_buffer_free (vm, &bi0, 1);
2137                       continue;
2138                     }
2139
2140                   /* Copy in destination address we are requesting from the
2141                      incomplete adj */
2142                   h0->ip4_over_ethernet[1].ip4.data_u32 =
2143                     adj0->sub_type.nbr.next_hop.ip4.as_u32;
2144                 }
2145
2146               vlib_buffer_copy_trace_flag (vm, p0, bi0);
2147               b0 = vlib_get_buffer (vm, bi0);
2148               VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
2149               vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
2150
2151               vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2152
2153               vlib_set_next_frame_buffer (vm, node,
2154                                           adj0->rewrite_header.next_index,
2155                                           bi0);
2156             }
2157         }
2158
2159       vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
2160     }
2161
2162   return frame->n_vectors;
2163 }
2164
2165 static uword
2166 ip4_arp (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
2167 {
2168   return (ip4_arp_inline (vm, node, frame, 0));
2169 }
2170
2171 static uword
2172 ip4_glean (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
2173 {
2174   return (ip4_arp_inline (vm, node, frame, 1));
2175 }
2176
2177 static char *ip4_arp_error_strings[] = {
2178   [IP4_ARP_ERROR_DROP] = "address overflow drops",
2179   [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
2180   [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
2181   [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
2182   [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
2183   [IP4_ARP_ERROR_NO_SOURCE_ADDRESS] = "no source address for ARP request",
2184 };
2185
2186 VLIB_REGISTER_NODE (ip4_arp_node) =
2187 {
2188   .function = ip4_arp,.name = "ip4-arp",.vector_size =
2189     sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_errors =
2190     ARRAY_LEN (ip4_arp_error_strings),.error_strings =
2191     ip4_arp_error_strings,.n_next_nodes = IP4_ARP_N_NEXT,.next_nodes =
2192   {
2193   [IP4_ARP_NEXT_DROP] = "error-drop",}
2194 ,};
2195
2196 VLIB_REGISTER_NODE (ip4_glean_node) =
2197 {
2198   .function = ip4_glean,.name = "ip4-glean",.vector_size =
2199     sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_errors =
2200     ARRAY_LEN (ip4_arp_error_strings),.error_strings =
2201     ip4_arp_error_strings,.n_next_nodes = IP4_ARP_N_NEXT,.next_nodes =
2202   {
2203   [IP4_ARP_NEXT_DROP] = "error-drop",}
2204 ,};
2205
2206 #define foreach_notrace_ip4_arp_error           \
2207 _(DROP)                                         \
2208 _(REQUEST_SENT)                                 \
2209 _(REPLICATE_DROP)                               \
2210 _(REPLICATE_FAIL)
2211
2212 clib_error_t *
2213 arp_notrace_init (vlib_main_t * vm)
2214 {
2215   vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, ip4_arp_node.index);
2216
2217   /* don't trace ARP request packets */
2218 #define _(a)                                    \
2219     vnet_pcap_drop_trace_filter_add_del         \
2220         (rt->errors[IP4_ARP_ERROR_##a],         \
2221          1 /* is_add */);
2222   foreach_notrace_ip4_arp_error;
2223 #undef _
2224   return 0;
2225 }
2226
2227 VLIB_INIT_FUNCTION (arp_notrace_init);
2228
2229
2230 /* Send an ARP request to see if given destination is reachable on given interface. */
2231 clib_error_t *
2232 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
2233 {
2234   vnet_main_t *vnm = vnet_get_main ();
2235   ip4_main_t *im = &ip4_main;
2236   ethernet_arp_header_t *h;
2237   ip4_address_t *src;
2238   ip_interface_address_t *ia;
2239   ip_adjacency_t *adj;
2240   vnet_hw_interface_t *hi;
2241   vnet_sw_interface_t *si;
2242   vlib_buffer_t *b;
2243   adj_index_t ai;
2244   u32 bi = 0;
2245
2246   si = vnet_get_sw_interface (vnm, sw_if_index);
2247
2248   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2249     {
2250       return clib_error_return (0, "%U: interface %U down",
2251                                 format_ip4_address, dst,
2252                                 format_vnet_sw_if_index_name, vnm,
2253                                 sw_if_index);
2254     }
2255
2256   src =
2257     ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2258   if (!src)
2259     {
2260       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2261       return clib_error_return
2262         (0,
2263          "no matching interface address for destination %U (interface %U)",
2264          format_ip4_address, dst, format_vnet_sw_if_index_name, vnm,
2265          sw_if_index);
2266     }
2267
2268   h = vlib_packet_template_get_packet (vm,
2269                                        &im->ip4_arp_request_packet_template,
2270                                        &bi);
2271
2272   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2273   if (PREDICT_FALSE (!hi->hw_address))
2274     {
2275       return clib_error_return (0, "%U: interface %U do not support ip probe",
2276                                 format_ip4_address, dst,
2277                                 format_vnet_sw_if_index_name, vnm,
2278                                 sw_if_index);
2279     }
2280
2281   clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address,
2282                sizeof (h->ip4_over_ethernet[0].ethernet));
2283
2284   h->ip4_over_ethernet[0].ip4 = src[0];
2285   h->ip4_over_ethernet[1].ip4 = dst[0];
2286
2287   b = vlib_get_buffer (vm, bi);
2288   vnet_buffer (b)->sw_if_index[VLIB_RX] =
2289     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2290
2291   ip46_address_t nh = {
2292     .ip4 = *dst,
2293   };
2294
2295   ai = adj_nbr_add_or_lock (FIB_PROTOCOL_IP4,
2296                             VNET_LINK_IP4, &nh, sw_if_index);
2297   adj = adj_get (ai);
2298
2299   /* Peer has been previously resolved, retrieve glean adj instead */
2300   if (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE)
2301     {
2302       adj_unlock (ai);
2303       ai = adj_glean_add_or_lock (FIB_PROTOCOL_IP4, sw_if_index, &nh);
2304       adj = adj_get (ai);
2305     }
2306
2307   /* Add encapsulation string for software interface (e.g. ethernet header). */
2308   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2309   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2310
2311   {
2312     vlib_frame_t *f = vlib_get_frame_to_node (vm, hi->output_node_index);
2313     u32 *to_next = vlib_frame_vector_args (f);
2314     to_next[0] = bi;
2315     f->n_vectors = 1;
2316     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2317   }
2318
2319   adj_unlock (ai);
2320   return /* no error */ 0;
2321 }
2322
2323 typedef enum
2324 {
2325   IP4_REWRITE_NEXT_DROP,
2326   IP4_REWRITE_NEXT_ICMP_ERROR,
2327 } ip4_rewrite_next_t;
2328
2329 always_inline uword
2330 ip4_rewrite_inline (vlib_main_t * vm,
2331                     vlib_node_runtime_t * node,
2332                     vlib_frame_t * frame,
2333                     int do_counters, int is_midchain, int is_mcast)
2334 {
2335   ip_lookup_main_t *lm = &ip4_main.lookup_main;
2336   u32 *from = vlib_frame_vector_args (frame);
2337   u32 n_left_from, n_left_to_next, *to_next, next_index;
2338   vlib_node_runtime_t *error_node =
2339     vlib_node_get_runtime (vm, ip4_input_node.index);
2340
2341   n_left_from = frame->n_vectors;
2342   next_index = node->cached_next_index;
2343   u32 thread_index = vlib_get_thread_index ();
2344
2345   while (n_left_from > 0)
2346     {
2347       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2348
2349       while (n_left_from >= 4 && n_left_to_next >= 2)
2350         {
2351           ip_adjacency_t *adj0, *adj1;
2352           vlib_buffer_t *p0, *p1;
2353           ip4_header_t *ip0, *ip1;
2354           u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
2355           u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
2356           u32 tx_sw_if_index0, tx_sw_if_index1;
2357
2358           /* Prefetch next iteration. */
2359           {
2360             vlib_buffer_t *p2, *p3;
2361
2362             p2 = vlib_get_buffer (vm, from[2]);
2363             p3 = vlib_get_buffer (vm, from[3]);
2364
2365             vlib_prefetch_buffer_header (p2, STORE);
2366             vlib_prefetch_buffer_header (p3, STORE);
2367
2368             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2369             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2370           }
2371
2372           pi0 = to_next[0] = from[0];
2373           pi1 = to_next[1] = from[1];
2374
2375           from += 2;
2376           n_left_from -= 2;
2377           to_next += 2;
2378           n_left_to_next -= 2;
2379
2380           p0 = vlib_get_buffer (vm, pi0);
2381           p1 = vlib_get_buffer (vm, pi1);
2382
2383           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2384           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
2385
2386           /*
2387            * pre-fetch the per-adjacency counters
2388            */
2389           if (do_counters)
2390             {
2391               vlib_prefetch_combined_counter (&adjacency_counters,
2392                                               thread_index, adj_index0);
2393               vlib_prefetch_combined_counter (&adjacency_counters,
2394                                               thread_index, adj_index1);
2395             }
2396
2397           ip0 = vlib_buffer_get_current (p0);
2398           ip1 = vlib_buffer_get_current (p1);
2399
2400           error0 = error1 = IP4_ERROR_NONE;
2401           next0 = next1 = IP4_REWRITE_NEXT_DROP;
2402
2403           /* Decrement TTL & update checksum.
2404              Works either endian, so no need for byte swap. */
2405           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2406             {
2407               i32 ttl0 = ip0->ttl;
2408
2409               /* Input node should have reject packets with ttl 0. */
2410               ASSERT (ip0->ttl > 0);
2411
2412               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2413               checksum0 += checksum0 >= 0xffff;
2414
2415               ip0->checksum = checksum0;
2416               ttl0 -= 1;
2417               ip0->ttl = ttl0;
2418
2419               /*
2420                * If the ttl drops below 1 when forwarding, generate
2421                * an ICMP response.
2422                */
2423               if (PREDICT_FALSE (ttl0 <= 0))
2424                 {
2425                   error0 = IP4_ERROR_TIME_EXPIRED;
2426                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2427                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2428                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2429                                                0);
2430                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2431                 }
2432
2433               /* Verify checksum. */
2434               ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
2435                       (p0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2436             }
2437           else
2438             {
2439               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2440             }
2441           if (PREDICT_TRUE (!(p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2442             {
2443               i32 ttl1 = ip1->ttl;
2444
2445               /* Input node should have reject packets with ttl 0. */
2446               ASSERT (ip1->ttl > 0);
2447
2448               checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2449               checksum1 += checksum1 >= 0xffff;
2450
2451               ip1->checksum = checksum1;
2452               ttl1 -= 1;
2453               ip1->ttl = ttl1;
2454
2455               /*
2456                * If the ttl drops below 1 when forwarding, generate
2457                * an ICMP response.
2458                */
2459               if (PREDICT_FALSE (ttl1 <= 0))
2460                 {
2461                   error1 = IP4_ERROR_TIME_EXPIRED;
2462                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2463                   icmp4_error_set_vnet_buffer (p1, ICMP4_time_exceeded,
2464                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2465                                                0);
2466                   next1 = IP4_REWRITE_NEXT_ICMP_ERROR;
2467                 }
2468
2469               /* Verify checksum. */
2470               ASSERT ((ip1->checksum == ip4_header_checksum (ip1)) ||
2471                       (p1->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2472             }
2473           else
2474             {
2475               p1->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2476             }
2477
2478           /* Rewrite packet header and updates lengths. */
2479           adj0 = adj_get (adj_index0);
2480           adj1 = adj_get (adj_index1);
2481
2482           /* Worth pipelining. No guarantee that adj0,1 are hot... */
2483           rw_len0 = adj0[0].rewrite_header.data_bytes;
2484           rw_len1 = adj1[0].rewrite_header.data_bytes;
2485           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2486           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
2487
2488           /* Check MTU of outgoing interface. */
2489           error0 =
2490             (vlib_buffer_length_in_chain (vm, p0) >
2491              adj0[0].
2492              rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
2493              error0);
2494           error1 =
2495             (vlib_buffer_length_in_chain (vm, p1) >
2496              adj1[0].
2497              rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
2498              error1);
2499
2500           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2501            * to see the IP headerr */
2502           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2503             {
2504               next0 = adj0[0].rewrite_header.next_index;
2505               p0->current_data -= rw_len0;
2506               p0->current_length += rw_len0;
2507               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2508               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2509
2510               if (PREDICT_FALSE
2511                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2512                 vnet_feature_arc_start (lm->output_feature_arc_index,
2513                                         tx_sw_if_index0, &next0, p0);
2514             }
2515           if (PREDICT_TRUE (error1 == IP4_ERROR_NONE))
2516             {
2517               next1 = adj1[0].rewrite_header.next_index;
2518               p1->current_data -= rw_len1;
2519               p1->current_length += rw_len1;
2520
2521               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
2522               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
2523
2524               if (PREDICT_FALSE
2525                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2526                 vnet_feature_arc_start (lm->output_feature_arc_index,
2527                                         tx_sw_if_index1, &next1, p1);
2528             }
2529
2530           /* Guess we are only writing on simple Ethernet header. */
2531           vnet_rewrite_two_headers (adj0[0], adj1[0],
2532                                     ip0, ip1, sizeof (ethernet_header_t));
2533
2534           /*
2535            * Bump the per-adjacency counters
2536            */
2537           if (do_counters)
2538             {
2539               vlib_increment_combined_counter
2540                 (&adjacency_counters,
2541                  thread_index,
2542                  adj_index0, 1,
2543                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2544
2545               vlib_increment_combined_counter
2546                 (&adjacency_counters,
2547                  thread_index,
2548                  adj_index1, 1,
2549                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
2550             }
2551
2552           if (is_midchain)
2553             {
2554               adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
2555               adj1->sub_type.midchain.fixup_func (vm, adj1, p1);
2556             }
2557           if (is_mcast)
2558             {
2559               /*
2560                * copy bytes from the IP address into the MAC rewrite
2561                */
2562               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2563               vnet_fixup_one_header (adj1[0], &ip1->dst_address, ip1);
2564             }
2565
2566           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2567                                            to_next, n_left_to_next,
2568                                            pi0, pi1, next0, next1);
2569         }
2570
2571       while (n_left_from > 0 && n_left_to_next > 0)
2572         {
2573           ip_adjacency_t *adj0;
2574           vlib_buffer_t *p0;
2575           ip4_header_t *ip0;
2576           u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2577           u32 tx_sw_if_index0;
2578
2579           pi0 = to_next[0] = from[0];
2580
2581           p0 = vlib_get_buffer (vm, pi0);
2582
2583           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2584
2585           adj0 = adj_get (adj_index0);
2586
2587           ip0 = vlib_buffer_get_current (p0);
2588
2589           error0 = IP4_ERROR_NONE;
2590           next0 = IP4_REWRITE_NEXT_DROP;        /* drop on error */
2591
2592           /* Decrement TTL & update checksum. */
2593           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2594             {
2595               i32 ttl0 = ip0->ttl;
2596
2597               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2598
2599               checksum0 += checksum0 >= 0xffff;
2600
2601               ip0->checksum = checksum0;
2602
2603               ASSERT (ip0->ttl > 0);
2604
2605               ttl0 -= 1;
2606
2607               ip0->ttl = ttl0;
2608
2609               ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
2610                       (p0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2611
2612               if (PREDICT_FALSE (ttl0 <= 0))
2613                 {
2614                   /*
2615                    * If the ttl drops below 1 when forwarding, generate
2616                    * an ICMP response.
2617                    */
2618                   error0 = IP4_ERROR_TIME_EXPIRED;
2619                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2620                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2621                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2622                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2623                                                0);
2624                 }
2625             }
2626           else
2627             {
2628               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2629             }
2630
2631           if (do_counters)
2632             vlib_prefetch_combined_counter (&adjacency_counters,
2633                                             thread_index, adj_index0);
2634
2635           /* Guess we are only writing on simple Ethernet header. */
2636           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2637           if (is_mcast)
2638             {
2639               /*
2640                * copy bytes from the IP address into the MAC rewrite
2641                */
2642               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2643             }
2644
2645           /* Update packet buffer attributes/set output interface. */
2646           rw_len0 = adj0[0].rewrite_header.data_bytes;
2647           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2648
2649           if (do_counters)
2650             vlib_increment_combined_counter
2651               (&adjacency_counters,
2652                thread_index, adj_index0, 1,
2653                vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2654
2655           /* Check MTU of outgoing interface. */
2656           error0 = (vlib_buffer_length_in_chain (vm, p0)
2657                     > adj0[0].rewrite_header.max_l3_packet_bytes
2658                     ? IP4_ERROR_MTU_EXCEEDED : error0);
2659
2660           p0->error = error_node->errors[error0];
2661
2662           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2663            * to see the IP headerr */
2664           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2665             {
2666               p0->current_data -= rw_len0;
2667               p0->current_length += rw_len0;
2668               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2669
2670               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2671               next0 = adj0[0].rewrite_header.next_index;
2672
2673               if (is_midchain)
2674                 {
2675                   adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
2676                 }
2677
2678               if (PREDICT_FALSE
2679                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2680                 vnet_feature_arc_start (lm->output_feature_arc_index,
2681                                         tx_sw_if_index0, &next0, p0);
2682
2683             }
2684
2685           from += 1;
2686           n_left_from -= 1;
2687           to_next += 1;
2688           n_left_to_next -= 1;
2689
2690           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2691                                            to_next, n_left_to_next,
2692                                            pi0, next0);
2693         }
2694
2695       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2696     }
2697
2698   /* Need to do trace after rewrites to pick up new packet data. */
2699   if (node->flags & VLIB_NODE_FLAG_TRACE)
2700     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2701
2702   return frame->n_vectors;
2703 }
2704
2705
2706 /** @brief IPv4 rewrite node.
2707     @node ip4-rewrite
2708
2709     This is the IPv4 transit-rewrite node: decrement TTL, fix the ipv4
2710     header checksum, fetch the ip adjacency, check the outbound mtu,
2711     apply the adjacency rewrite, and send pkts to the adjacency
2712     rewrite header's rewrite_next_index.
2713
2714     @param vm vlib_main_t corresponding to the current thread
2715     @param node vlib_node_runtime_t
2716     @param frame vlib_frame_t whose contents should be dispatched
2717
2718     @par Graph mechanics: buffer metadata, next index usage
2719
2720     @em Uses:
2721     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
2722         - the rewrite adjacency index
2723     - <code>adj->lookup_next_index</code>
2724         - Must be IP_LOOKUP_NEXT_REWRITE or IP_LOOKUP_NEXT_ARP, otherwise
2725           the packet will be dropped.
2726     - <code>adj->rewrite_header</code>
2727         - Rewrite string length, rewrite string, next_index
2728
2729     @em Sets:
2730     - <code>b->current_data, b->current_length</code>
2731         - Updated net of applying the rewrite string
2732
2733     <em>Next Indices:</em>
2734     - <code> adj->rewrite_header.next_index </code>
2735       or @c error-drop
2736 */
2737 static uword
2738 ip4_rewrite (vlib_main_t * vm,
2739              vlib_node_runtime_t * node, vlib_frame_t * frame)
2740 {
2741   if (adj_are_counters_enabled ())
2742     return ip4_rewrite_inline (vm, node, frame, 1, 0, 0);
2743   else
2744     return ip4_rewrite_inline (vm, node, frame, 0, 0, 0);
2745 }
2746
2747 static uword
2748 ip4_midchain (vlib_main_t * vm,
2749               vlib_node_runtime_t * node, vlib_frame_t * frame)
2750 {
2751   if (adj_are_counters_enabled ())
2752     return ip4_rewrite_inline (vm, node, frame, 1, 1, 0);
2753   else
2754     return ip4_rewrite_inline (vm, node, frame, 0, 1, 0);
2755 }
2756
2757 static uword
2758 ip4_rewrite_mcast (vlib_main_t * vm,
2759                    vlib_node_runtime_t * node, vlib_frame_t * frame)
2760 {
2761   if (adj_are_counters_enabled ())
2762     return ip4_rewrite_inline (vm, node, frame, 1, 0, 1);
2763   else
2764     return ip4_rewrite_inline (vm, node, frame, 0, 0, 1);
2765 }
2766
2767 static uword
2768 ip4_mcast_midchain (vlib_main_t * vm,
2769                     vlib_node_runtime_t * node, vlib_frame_t * frame)
2770 {
2771   if (adj_are_counters_enabled ())
2772     return ip4_rewrite_inline (vm, node, frame, 1, 1, 1);
2773   else
2774     return ip4_rewrite_inline (vm, node, frame, 0, 1, 1);
2775 }
2776
2777 /* *INDENT-OFF* */
2778 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2779   .function = ip4_rewrite,
2780   .name = "ip4-rewrite",
2781   .vector_size = sizeof (u32),
2782
2783   .format_trace = format_ip4_rewrite_trace,
2784
2785   .n_next_nodes = 2,
2786   .next_nodes = {
2787     [IP4_REWRITE_NEXT_DROP] = "error-drop",
2788     [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2789   },
2790 };
2791 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite)
2792
2793 VLIB_REGISTER_NODE (ip4_rewrite_mcast_node) = {
2794   .function = ip4_rewrite_mcast,
2795   .name = "ip4-rewrite-mcast",
2796   .vector_size = sizeof (u32),
2797
2798   .format_trace = format_ip4_rewrite_trace,
2799   .sibling_of = "ip4-rewrite",
2800 };
2801 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_mcast_node, ip4_rewrite_mcast)
2802
2803 VLIB_REGISTER_NODE (ip4_mcast_midchain_node, static) = {
2804   .function = ip4_mcast_midchain,
2805   .name = "ip4-mcast-midchain",
2806   .vector_size = sizeof (u32),
2807
2808   .format_trace = format_ip4_rewrite_trace,
2809   .sibling_of = "ip4-rewrite",
2810 };
2811 VLIB_NODE_FUNCTION_MULTIARCH (ip4_mcast_midchain_node, ip4_mcast_midchain)
2812
2813 VLIB_REGISTER_NODE (ip4_midchain_node) = {
2814   .function = ip4_midchain,
2815   .name = "ip4-midchain",
2816   .vector_size = sizeof (u32),
2817   .format_trace = format_ip4_forward_next_trace,
2818   .sibling_of =  "ip4-rewrite",
2819 };
2820 VLIB_NODE_FUNCTION_MULTIARCH (ip4_midchain_node, ip4_midchain);
2821 /* *INDENT-ON */
2822
2823 int
2824 ip4_lookup_validate (ip4_address_t * a, u32 fib_index0)
2825 {
2826   ip4_fib_mtrie_t *mtrie0;
2827   ip4_fib_mtrie_leaf_t leaf0;
2828   u32 lbi0;
2829
2830   mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
2831
2832   leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, a);
2833   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
2834   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
2835
2836   lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2837
2838   return lbi0 == ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0), a);
2839 }
2840
2841 static clib_error_t *
2842 test_lookup_command_fn (vlib_main_t * vm,
2843                         unformat_input_t * input, vlib_cli_command_t * cmd)
2844 {
2845   ip4_fib_t *fib;
2846   u32 table_id = 0;
2847   f64 count = 1;
2848   u32 n;
2849   int i;
2850   ip4_address_t ip4_base_address;
2851   u64 errors = 0;
2852
2853   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2854     {
2855       if (unformat (input, "table %d", &table_id))
2856         {
2857           /* Make sure the entry exists. */
2858           fib = ip4_fib_get (table_id);
2859           if ((fib) && (fib->index != table_id))
2860             return clib_error_return (0, "<fib-index> %d does not exist",
2861                                       table_id);
2862         }
2863       else if (unformat (input, "count %f", &count))
2864         ;
2865
2866       else if (unformat (input, "%U",
2867                          unformat_ip4_address, &ip4_base_address))
2868         ;
2869       else
2870         return clib_error_return (0, "unknown input `%U'",
2871                                   format_unformat_error, input);
2872     }
2873
2874   n = count;
2875
2876   for (i = 0; i < n; i++)
2877     {
2878       if (!ip4_lookup_validate (&ip4_base_address, table_id))
2879         errors++;
2880
2881       ip4_base_address.as_u32 =
2882         clib_host_to_net_u32 (1 +
2883                               clib_net_to_host_u32 (ip4_base_address.as_u32));
2884     }
2885
2886   if (errors)
2887     vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
2888   else
2889     vlib_cli_output (vm, "No errors in %d lookups\n", n);
2890
2891   return 0;
2892 }
2893
2894 /*?
2895  * Perform a lookup of an IPv4 Address (or range of addresses) in the
2896  * given FIB table to determine if there is a conflict with the
2897  * adjacency table. The fib-id can be determined by using the
2898  * '<em>show ip fib</em>' command. If fib-id is not entered, default value
2899  * of 0 is used.
2900  *
2901  * @todo This command uses fib-id, other commands use table-id (not
2902  * just a name, they are different indexes). Would like to change this
2903  * to table-id for consistency.
2904  *
2905  * @cliexpar
2906  * Example of how to run the test lookup command:
2907  * @cliexstart{test lookup 172.16.1.1 table 1 count 2}
2908  * No errors in 2 lookups
2909  * @cliexend
2910 ?*/
2911 /* *INDENT-OFF* */
2912 VLIB_CLI_COMMAND (lookup_test_command, static) =
2913 {
2914   .path = "test lookup",
2915   .short_help = "test lookup <ipv4-addr> [table <fib-id>] [count <nn>]",
2916   .function = test_lookup_command_fn,
2917 };
2918 /* *INDENT-ON* */
2919
2920 int
2921 vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
2922 {
2923   u32 fib_index;
2924
2925   fib_index = fib_table_find (FIB_PROTOCOL_IP4, table_id);
2926
2927   if (~0 == fib_index)
2928     return VNET_API_ERROR_NO_SUCH_FIB;
2929
2930   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP4,
2931                                   flow_hash_config);
2932
2933   return 0;
2934 }
2935
2936 static clib_error_t *
2937 set_ip_flow_hash_command_fn (vlib_main_t * vm,
2938                              unformat_input_t * input,
2939                              vlib_cli_command_t * cmd)
2940 {
2941   int matched = 0;
2942   u32 table_id = 0;
2943   u32 flow_hash_config = 0;
2944   int rv;
2945
2946   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2947     {
2948       if (unformat (input, "table %d", &table_id))
2949         matched = 1;
2950 #define _(a,v) \
2951     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2952       foreach_flow_hash_bit
2953 #undef _
2954         else
2955         break;
2956     }
2957
2958   if (matched == 0)
2959     return clib_error_return (0, "unknown input `%U'",
2960                               format_unformat_error, input);
2961
2962   rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
2963   switch (rv)
2964     {
2965     case 0:
2966       break;
2967
2968     case VNET_API_ERROR_NO_SUCH_FIB:
2969       return clib_error_return (0, "no such FIB table %d", table_id);
2970
2971     default:
2972       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2973       break;
2974     }
2975
2976   return 0;
2977 }
2978
2979 /*?
2980  * Configure the set of IPv4 fields used by the flow hash.
2981  *
2982  * @cliexpar
2983  * Example of how to set the flow hash on a given table:
2984  * @cliexcmd{set ip flow-hash table 7 dst sport dport proto}
2985  * Example of display the configured flow hash:
2986  * @cliexstart{show ip fib}
2987  * ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2988  * 0.0.0.0/0
2989  *   unicast-ip4-chain
2990  *   [@0]: dpo-load-balance: [index:0 buckets:1 uRPF:0 to:[0:0]]
2991  *     [0] [@0]: dpo-drop ip6
2992  * 0.0.0.0/32
2993  *   unicast-ip4-chain
2994  *   [@0]: dpo-load-balance: [index:1 buckets:1 uRPF:1 to:[0:0]]
2995  *     [0] [@0]: dpo-drop ip6
2996  * 224.0.0.0/8
2997  *   unicast-ip4-chain
2998  *   [@0]: dpo-load-balance: [index:3 buckets:1 uRPF:3 to:[0:0]]
2999  *     [0] [@0]: dpo-drop ip6
3000  * 6.0.1.2/32
3001  *   unicast-ip4-chain
3002  *   [@0]: dpo-load-balance: [index:30 buckets:1 uRPF:29 to:[0:0]]
3003  *     [0] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
3004  * 7.0.0.1/32
3005  *   unicast-ip4-chain
3006  *   [@0]: dpo-load-balance: [index:31 buckets:4 uRPF:30 to:[0:0]]
3007  *     [0] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3008  *     [1] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3009  *     [2] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3010  *     [3] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
3011  * 240.0.0.0/8
3012  *   unicast-ip4-chain
3013  *   [@0]: dpo-load-balance: [index:2 buckets:1 uRPF:2 to:[0:0]]
3014  *     [0] [@0]: dpo-drop ip6
3015  * 255.255.255.255/32
3016  *   unicast-ip4-chain
3017  *   [@0]: dpo-load-balance: [index:4 buckets:1 uRPF:4 to:[0:0]]
3018  *     [0] [@0]: dpo-drop ip6
3019  * ipv4-VRF:7, fib_index 1, flow hash: dst sport dport proto
3020  * 0.0.0.0/0
3021  *   unicast-ip4-chain
3022  *   [@0]: dpo-load-balance: [index:12 buckets:1 uRPF:11 to:[0:0]]
3023  *     [0] [@0]: dpo-drop ip6
3024  * 0.0.0.0/32
3025  *   unicast-ip4-chain
3026  *   [@0]: dpo-load-balance: [index:13 buckets:1 uRPF:12 to:[0:0]]
3027  *     [0] [@0]: dpo-drop ip6
3028  * 172.16.1.0/24
3029  *   unicast-ip4-chain
3030  *   [@0]: dpo-load-balance: [index:17 buckets:1 uRPF:16 to:[0:0]]
3031  *     [0] [@4]: ipv4-glean: af_packet0
3032  * 172.16.1.1/32
3033  *   unicast-ip4-chain
3034  *   [@0]: dpo-load-balance: [index:18 buckets:1 uRPF:17 to:[1:84]]
3035  *     [0] [@2]: dpo-receive: 172.16.1.1 on af_packet0
3036  * 172.16.1.2/32
3037  *   unicast-ip4-chain
3038  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
3039  *     [0] [@5]: ipv4 via 172.16.1.2 af_packet0: IP4: 02:fe:9e:70:7a:2b -> 26:a5:f6:9c:3a:36
3040  * 172.16.2.0/24
3041  *   unicast-ip4-chain
3042  *   [@0]: dpo-load-balance: [index:19 buckets:1 uRPF:18 to:[0:0]]
3043  *     [0] [@4]: ipv4-glean: af_packet1
3044  * 172.16.2.1/32
3045  *   unicast-ip4-chain
3046  *   [@0]: dpo-load-balance: [index:20 buckets:1 uRPF:19 to:[0:0]]
3047  *     [0] [@2]: dpo-receive: 172.16.2.1 on af_packet1
3048  * 224.0.0.0/8
3049  *   unicast-ip4-chain
3050  *   [@0]: dpo-load-balance: [index:15 buckets:1 uRPF:14 to:[0:0]]
3051  *     [0] [@0]: dpo-drop ip6
3052  * 240.0.0.0/8
3053  *   unicast-ip4-chain
3054  *   [@0]: dpo-load-balance: [index:14 buckets:1 uRPF:13 to:[0:0]]
3055  *     [0] [@0]: dpo-drop ip6
3056  * 255.255.255.255/32
3057  *   unicast-ip4-chain
3058  *   [@0]: dpo-load-balance: [index:16 buckets:1 uRPF:15 to:[0:0]]
3059  *     [0] [@0]: dpo-drop ip6
3060  * @cliexend
3061 ?*/
3062 /* *INDENT-OFF* */
3063 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) =
3064 {
3065   .path = "set ip flow-hash",
3066   .short_help =
3067   "set ip flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
3068   .function = set_ip_flow_hash_command_fn,
3069 };
3070 /* *INDENT-ON* */
3071
3072 int
3073 vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3074                              u32 table_index)
3075 {
3076   vnet_main_t *vnm = vnet_get_main ();
3077   vnet_interface_main_t *im = &vnm->interface_main;
3078   ip4_main_t *ipm = &ip4_main;
3079   ip_lookup_main_t *lm = &ipm->lookup_main;
3080   vnet_classify_main_t *cm = &vnet_classify_main;
3081   ip4_address_t *if_addr;
3082
3083   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3084     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3085
3086   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3087     return VNET_API_ERROR_NO_SUCH_ENTRY;
3088
3089   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3090   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
3091
3092   if_addr = ip4_interface_first_address (ipm, sw_if_index, NULL);
3093
3094   if (NULL != if_addr)
3095     {
3096       fib_prefix_t pfx = {
3097         .fp_len = 32,
3098         .fp_proto = FIB_PROTOCOL_IP4,
3099         .fp_addr.ip4 = *if_addr,
3100       };
3101       u32 fib_index;
3102
3103       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3104                                                        sw_if_index);
3105
3106
3107       if (table_index != (u32) ~ 0)
3108         {
3109           dpo_id_t dpo = DPO_INVALID;
3110
3111           dpo_set (&dpo,
3112                    DPO_CLASSIFY,
3113                    DPO_PROTO_IP4,
3114                    classify_dpo_create (DPO_PROTO_IP4, table_index));
3115
3116           fib_table_entry_special_dpo_add (fib_index,
3117                                            &pfx,
3118                                            FIB_SOURCE_CLASSIFY,
3119                                            FIB_ENTRY_FLAG_NONE, &dpo);
3120           dpo_reset (&dpo);
3121         }
3122       else
3123         {
3124           fib_table_entry_special_remove (fib_index,
3125                                           &pfx, FIB_SOURCE_CLASSIFY);
3126         }
3127     }
3128
3129   return 0;
3130 }
3131
3132 static clib_error_t *
3133 set_ip_classify_command_fn (vlib_main_t * vm,
3134                             unformat_input_t * input,
3135                             vlib_cli_command_t * cmd)
3136 {
3137   u32 table_index = ~0;
3138   int table_index_set = 0;
3139   u32 sw_if_index = ~0;
3140   int rv;
3141
3142   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3143     {
3144       if (unformat (input, "table-index %d", &table_index))
3145         table_index_set = 1;
3146       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3147                          vnet_get_main (), &sw_if_index))
3148         ;
3149       else
3150         break;
3151     }
3152
3153   if (table_index_set == 0)
3154     return clib_error_return (0, "classify table-index must be specified");
3155
3156   if (sw_if_index == ~0)
3157     return clib_error_return (0, "interface / subif must be specified");
3158
3159   rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
3160
3161   switch (rv)
3162     {
3163     case 0:
3164       break;
3165
3166     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3167       return clib_error_return (0, "No such interface");
3168
3169     case VNET_API_ERROR_NO_SUCH_ENTRY:
3170       return clib_error_return (0, "No such classifier table");
3171     }
3172   return 0;
3173 }
3174
3175 /*?
3176  * Assign a classification table to an interface. The classification
3177  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3178  * commands. Once the table is create, use this command to filter packets
3179  * on an interface.
3180  *
3181  * @cliexpar
3182  * Example of how to assign a classification table to an interface:
3183  * @cliexcmd{set ip classify intfc GigabitEthernet2/0/0 table-index 1}
3184 ?*/
3185 /* *INDENT-OFF* */
3186 VLIB_CLI_COMMAND (set_ip_classify_command, static) =
3187 {
3188     .path = "set ip classify",
3189     .short_help =
3190     "set ip classify intfc <interface> table-index <classify-idx>",
3191     .function = set_ip_classify_command_fn,
3192 };
3193 /* *INDENT-ON* */
3194
3195 /*
3196  * fd.io coding-style-patch-verification: ON
3197  *
3198  * Local Variables:
3199  * eval: (c-set-style "gnu")
3200  * End:
3201  */