src/vnet/ip/ip4_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip4_forward.c: IP v4 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  43 #include <vnet/ethernet/arp_packet.h>   /* for ethernet_arp_header_t */
  44 #include <vnet/ppp/ppp.h>
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vnet/api_errno.h>     /* for API error numbers */
  47 #include <vnet/fib/fib_table.h> /* for FIB table and entry creation */
  48 #include <vnet/fib/fib_entry.h> /* for FIB table and entry creation */
  49 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  50 #include <vnet/fib/ip4_fib.h>
  51 #include <vnet/dpo/load_balance.h>
  52 #include <vnet/dpo/load_balance_map.h>
  53 #include <vnet/dpo/classify_dpo.h>
  54 #include <vnet/mfib/mfib_table.h>       /* for mFIB table and entry creation */
  55
  56 /**
  57  * @file
  58  * @brief IPv4 Forwarding.
  59  *
  60  * This file contains the source code for IPv4 forwarding.
  61  */
  62
  63 void
  64 ip4_forward_next_trace (vlib_main_t * vm,
  65                         vlib_node_runtime_t * node,
  66                         vlib_frame_t * frame,
  67                         vlib_rx_or_tx_t which_adj_index);
  68
  69 always_inline uword
  70 ip4_lookup_inline (vlib_main_t * vm,
  71                    vlib_node_runtime_t * node,
  72                    vlib_frame_t * frame,
  73                    int lookup_for_responses_to_locally_received_packets)
  74 {
  75   ip4_main_t *im = &ip4_main;
  76   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters;
  77   u32 n_left_from, n_left_to_next, *from, *to_next;
  78   ip_lookup_next_t next;
  79   u32 thread_index = vlib_get_thread_index ();
  80
  81   from = vlib_frame_vector_args (frame);
  82   n_left_from = frame->n_vectors;
  83   next = node->cached_next_index;
  84
  85   while (n_left_from > 0)
  86     {
  87       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
  88
  89       while (n_left_from >= 8 && n_left_to_next >= 4)
  90         {
  91           vlib_buffer_t *p0, *p1, *p2, *p3;
  92           ip4_header_t *ip0, *ip1, *ip2, *ip3;
  93           ip_lookup_next_t next0, next1, next2, next3;
  94           const load_balance_t *lb0, *lb1, *lb2, *lb3;
  95           ip4_fib_mtrie_t *mtrie0, *mtrie1, *mtrie2, *mtrie3;
  96           ip4_fib_mtrie_leaf_t leaf0, leaf1, leaf2, leaf3;
  97           ip4_address_t *dst_addr0, *dst_addr1, *dst_addr2, *dst_addr3;
  98           u32 pi0, fib_index0, lb_index0;
  99           u32 pi1, fib_index1, lb_index1;
 100           u32 pi2, fib_index2, lb_index2;
 101           u32 pi3, fib_index3, lb_index3;
 102           flow_hash_config_t flow_hash_config0, flow_hash_config1;
 103           flow_hash_config_t flow_hash_config2, flow_hash_config3;
 104           u32 hash_c0, hash_c1, hash_c2, hash_c3;
 105           const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3;
 106
 107           /* Prefetch next iteration. */
 108           {
 109             vlib_buffer_t *p4, *p5, *p6, *p7;
 110
 111             p4 = vlib_get_buffer (vm, from[4]);
 112             p5 = vlib_get_buffer (vm, from[5]);
 113             p6 = vlib_get_buffer (vm, from[6]);
 114             p7 = vlib_get_buffer (vm, from[7]);
 115
 116             vlib_prefetch_buffer_header (p4, LOAD);
 117             vlib_prefetch_buffer_header (p5, LOAD);
 118             vlib_prefetch_buffer_header (p6, LOAD);
 119             vlib_prefetch_buffer_header (p7, LOAD);
 120
 121             CLIB_PREFETCH (p4->data, sizeof (ip0[0]), LOAD);
 122             CLIB_PREFETCH (p5->data, sizeof (ip0[0]), LOAD);
 123             CLIB_PREFETCH (p6->data, sizeof (ip0[0]), LOAD);
 124             CLIB_PREFETCH (p7->data, sizeof (ip0[0]), LOAD);
 125           }
 126
 127           pi0 = to_next[0] = from[0];
 128           pi1 = to_next[1] = from[1];
 129           pi2 = to_next[2] = from[2];
 130           pi3 = to_next[3] = from[3];
 131
 132           from += 4;
 133           to_next += 4;
 134           n_left_to_next -= 4;
 135           n_left_from -= 4;
 136
 137           p0 = vlib_get_buffer (vm, pi0);
 138           p1 = vlib_get_buffer (vm, pi1);
 139           p2 = vlib_get_buffer (vm, pi2);
 140           p3 = vlib_get_buffer (vm, pi3);
 141
 142           ip0 = vlib_buffer_get_current (p0);
 143           ip1 = vlib_buffer_get_current (p1);
 144           ip2 = vlib_buffer_get_current (p2);
 145           ip3 = vlib_buffer_get_current (p3);
 146
 147           dst_addr0 = &ip0->dst_address;
 148           dst_addr1 = &ip1->dst_address;
 149           dst_addr2 = &ip2->dst_address;
 150           dst_addr3 = &ip3->dst_address;
 151
 152           fib_index0 =
 153             vec_elt (im->fib_index_by_sw_if_index,
 154                      vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 155           fib_index1 =
 156             vec_elt (im->fib_index_by_sw_if_index,
 157                      vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 158           fib_index2 =
 159             vec_elt (im->fib_index_by_sw_if_index,
 160                      vnet_buffer (p2)->sw_if_index[VLIB_RX]);
 161           fib_index3 =
 162             vec_elt (im->fib_index_by_sw_if_index,
 163                      vnet_buffer (p3)->sw_if_index[VLIB_RX]);
 164           fib_index0 =
 165             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
 166              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 167           fib_index1 =
 168             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
 169              (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
 170           fib_index2 =
 171             (vnet_buffer (p2)->sw_if_index[VLIB_TX] ==
 172              (u32) ~ 0) ? fib_index2 : vnet_buffer (p2)->sw_if_index[VLIB_TX];
 173           fib_index3 =
 174             (vnet_buffer (p3)->sw_if_index[VLIB_TX] ==
 175              (u32) ~ 0) ? fib_index3 : vnet_buffer (p3)->sw_if_index[VLIB_TX];
 176
 177
 178           if (!lookup_for_responses_to_locally_received_packets)
 179             {
 180               mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 181               mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
 182               mtrie2 = &ip4_fib_get (fib_index2)->mtrie;
 183               mtrie3 = &ip4_fib_get (fib_index3)->mtrie;
 184
 185               leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
 186               leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, dst_addr1);
 187               leaf2 = ip4_fib_mtrie_lookup_step_one (mtrie2, dst_addr2);
 188               leaf3 = ip4_fib_mtrie_lookup_step_one (mtrie3, dst_addr3);
 189             }
 190
 191           if (!lookup_for_responses_to_locally_received_packets)
 192             {
 193               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 194               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
 195               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 2);
 196               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 2);
 197             }
 198
 199           if (!lookup_for_responses_to_locally_received_packets)
 200             {
 201               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 202               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
 203               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 3);
 204               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 3);
 205             }
 206
 207           if (lookup_for_responses_to_locally_received_packets)
 208             {
 209               lb_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 210               lb_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
 211               lb_index2 = vnet_buffer (p2)->ip.adj_index[VLIB_RX];
 212               lb_index3 = vnet_buffer (p3)->ip.adj_index[VLIB_RX];
 213             }
 214           else
 215             {
 216               lb_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 217               lb_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 218               lb_index2 = ip4_fib_mtrie_leaf_get_adj_index (leaf2);
 219               lb_index3 = ip4_fib_mtrie_leaf_get_adj_index (leaf3);
 220             }
 221
 222           ASSERT (lb_index0 && lb_index1 && lb_index2 && lb_index3);
 223           lb0 = load_balance_get (lb_index0);
 224           lb1 = load_balance_get (lb_index1);
 225           lb2 = load_balance_get (lb_index2);
 226           lb3 = load_balance_get (lb_index3);
 227
 228           ASSERT (lb0->lb_n_buckets > 0);
 229           ASSERT (is_pow2 (lb0->lb_n_buckets));
 230           ASSERT (lb1->lb_n_buckets > 0);
 231           ASSERT (is_pow2 (lb1->lb_n_buckets));
 232           ASSERT (lb2->lb_n_buckets > 0);
 233           ASSERT (is_pow2 (lb2->lb_n_buckets));
 234           ASSERT (lb3->lb_n_buckets > 0);
 235           ASSERT (is_pow2 (lb3->lb_n_buckets));
 236
 237           /* Use flow hash to compute multipath adjacency. */
 238           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 239           hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
 240           hash_c2 = vnet_buffer (p2)->ip.flow_hash = 0;
 241           hash_c3 = vnet_buffer (p3)->ip.flow_hash = 0;
 242           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 243             {
 244               flow_hash_config0 = lb0->lb_hash_config;
 245               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 246                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 247               dpo0 =
 248                 load_balance_get_fwd_bucket (lb0,
 249                                              (hash_c0 &
 250                                               (lb0->lb_n_buckets_minus_1)));
 251             }
 252           else
 253             {
 254               dpo0 = load_balance_get_bucket_i (lb0, 0);
 255             }
 256           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 257             {
 258               flow_hash_config1 = lb1->lb_hash_config;
 259               hash_c1 = vnet_buffer (p1)->ip.flow_hash =
 260                 ip4_compute_flow_hash (ip1, flow_hash_config1);
 261               dpo1 =
 262                 load_balance_get_fwd_bucket (lb1,
 263                                              (hash_c1 &
 264                                               (lb1->lb_n_buckets_minus_1)));
 265             }
 266           else
 267             {
 268               dpo1 = load_balance_get_bucket_i (lb1, 0);
 269             }
 270           if (PREDICT_FALSE (lb2->lb_n_buckets > 1))
 271             {
 272               flow_hash_config2 = lb2->lb_hash_config;
 273               hash_c2 = vnet_buffer (p2)->ip.flow_hash =
 274                 ip4_compute_flow_hash (ip2, flow_hash_config2);
 275               dpo2 =
 276                 load_balance_get_fwd_bucket (lb2,
 277                                              (hash_c2 &
 278                                               (lb2->lb_n_buckets_minus_1)));
 279             }
 280           else
 281             {
 282               dpo2 = load_balance_get_bucket_i (lb2, 0);
 283             }
 284           if (PREDICT_FALSE (lb3->lb_n_buckets > 1))
 285             {
 286               flow_hash_config3 = lb3->lb_hash_config;
 287               hash_c3 = vnet_buffer (p3)->ip.flow_hash =
 288                 ip4_compute_flow_hash (ip3, flow_hash_config3);
 289               dpo3 =
 290                 load_balance_get_fwd_bucket (lb3,
 291                                              (hash_c3 &
 292                                               (lb3->lb_n_buckets_minus_1)));
 293             }
 294           else
 295             {
 296               dpo3 = load_balance_get_bucket_i (lb3, 0);
 297             }
 298
 299           next0 = dpo0->dpoi_next_node;
 300           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 301           next1 = dpo1->dpoi_next_node;
 302           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 303           next2 = dpo2->dpoi_next_node;
 304           vnet_buffer (p2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
 305           next3 = dpo3->dpoi_next_node;
 306           vnet_buffer (p3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
 307
 308           vlib_increment_combined_counter
 309             (cm, thread_index, lb_index0, 1,
 310              vlib_buffer_length_in_chain (vm, p0));
 311           vlib_increment_combined_counter
 312             (cm, thread_index, lb_index1, 1,
 313              vlib_buffer_length_in_chain (vm, p1));
 314           vlib_increment_combined_counter
 315             (cm, thread_index, lb_index2, 1,
 316              vlib_buffer_length_in_chain (vm, p2));
 317           vlib_increment_combined_counter
 318             (cm, thread_index, lb_index3, 1,
 319              vlib_buffer_length_in_chain (vm, p3));
 320
 321           vlib_validate_buffer_enqueue_x4 (vm, node, next,
 322                                            to_next, n_left_to_next,
 323                                            pi0, pi1, pi2, pi3,
 324                                            next0, next1, next2, next3);
 325         }
 326
 327       while (n_left_from > 0 && n_left_to_next > 0)
 328         {
 329           vlib_buffer_t *p0;
 330           ip4_header_t *ip0;
 331           ip_lookup_next_t next0;
 332           const load_balance_t *lb0;
 333           ip4_fib_mtrie_t *mtrie0;
 334           ip4_fib_mtrie_leaf_t leaf0;
 335           ip4_address_t *dst_addr0;
 336           u32 pi0, fib_index0, lbi0;
 337           flow_hash_config_t flow_hash_config0;
 338           const dpo_id_t *dpo0;
 339           u32 hash_c0;
 340
 341           pi0 = from[0];
 342           to_next[0] = pi0;
 343
 344           p0 = vlib_get_buffer (vm, pi0);
 345
 346           ip0 = vlib_buffer_get_current (p0);
 347
 348           dst_addr0 = &ip0->dst_address;
 349
 350           fib_index0 =
 351             vec_elt (im->fib_index_by_sw_if_index,
 352                      vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 353           fib_index0 =
 354             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
 355              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 356
 357           if (!lookup_for_responses_to_locally_received_packets)
 358             {
 359               mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 360
 361               leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
 362             }
 363
 364           if (!lookup_for_responses_to_locally_received_packets)
 365             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 366
 367           if (!lookup_for_responses_to_locally_received_packets)
 368             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 369
 370           if (lookup_for_responses_to_locally_received_packets)
 371             lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 372           else
 373             {
 374               /* Handle default route. */
 375               lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 376             }
 377
 378           ASSERT (lbi0);
 379           lb0 = load_balance_get (lbi0);
 380
 381           ASSERT (lb0->lb_n_buckets > 0);
 382           ASSERT (is_pow2 (lb0->lb_n_buckets));
 383
 384           /* Use flow hash to compute multipath adjacency. */
 385           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 386           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 387             {
 388               flow_hash_config0 = lb0->lb_hash_config;
 389
 390               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 391                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 392               dpo0 =
 393                 load_balance_get_fwd_bucket (lb0,
 394                                              (hash_c0 &
 395                                               (lb0->lb_n_buckets_minus_1)));
 396             }
 397           else
 398             {
 399               dpo0 = load_balance_get_bucket_i (lb0, 0);
 400             }
 401
 402           next0 = dpo0->dpoi_next_node;
 403           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 404
 405           vlib_increment_combined_counter (cm, thread_index, lbi0, 1,
 406                                            vlib_buffer_length_in_chain (vm,
 407                                                                         p0));
 408
 409           from += 1;
 410           to_next += 1;
 411           n_left_to_next -= 1;
 412           n_left_from -= 1;
 413
 414           if (PREDICT_FALSE (next0 != next))
 415             {
 416               n_left_to_next += 1;
 417               vlib_put_next_frame (vm, node, next, n_left_to_next);
 418               next = next0;
 419               vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 420               to_next[0] = pi0;
 421               to_next += 1;
 422               n_left_to_next -= 1;
 423             }
 424         }
 425
 426       vlib_put_next_frame (vm, node, next, n_left_to_next);
 427     }
 428
 429   if (node->flags & VLIB_NODE_FLAG_TRACE)
 430     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 431
 432   return frame->n_vectors;
 433 }
 434
 435 /** @brief IPv4 lookup node.
 436     @node ip4-lookup
 437
 438     This is the main IPv4 lookup dispatch node.
 439
 440     @param vm vlib_main_t corresponding to the current thread
 441     @param node vlib_node_runtime_t
 442     @param frame vlib_frame_t whose contents should be dispatched
 443
 444     @par Graph mechanics: buffer metadata, next index usage
 445
 446     @em Uses:
 447     - <code>vnet_buffer(b)->sw_if_index[VLIB_RX]</code>
 448         - Indicates the @c sw_if_index value of the interface that the
 449           packet was received on.
 450     - <code>vnet_buffer(b)->sw_if_index[VLIB_TX]</code>
 451         - When the value is @c ~0 then the node performs a longest prefix
 452           match (LPM) for the packet destination address in the FIB attached
 453           to the receive interface.
 454         - Otherwise perform LPM for the packet destination address in the
 455           indicated FIB. In this case <code>[VLIB_TX]</code> is a FIB index
 456           value (0, 1, ...) and not a VRF id.
 457
 458     @em Sets:
 459     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
 460         - The lookup result adjacency index.
 461
 462     <em>Next Index:</em>
 463     - Dispatches the packet to the node index found in
 464       ip_adjacency_t @c adj->lookup_next_index
 465       (where @c adj is the lookup result adjacency).
 466 */
 467 static uword
 468 ip4_lookup (vlib_main_t * vm,
 469             vlib_node_runtime_t * node, vlib_frame_t * frame)
 470 {
 471   return ip4_lookup_inline (vm, node, frame,
 472                             /* lookup_for_responses_to_locally_received_packets */
 473                             0);
 474
 475 }
 476
 477 static u8 *format_ip4_lookup_trace (u8 * s, va_list * args);
 478
 479 VLIB_REGISTER_NODE (ip4_lookup_node) =
 480 {
 481 .function = ip4_lookup,.name = "ip4-lookup",.vector_size =
 482     sizeof (u32),.format_trace = format_ip4_lookup_trace,.n_next_nodes =
 483     IP_LOOKUP_N_NEXT,.next_nodes = IP4_LOOKUP_NEXT_NODES,};
 484
 485 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup);
 486
 487 always_inline uword
 488 ip4_load_balance (vlib_main_t * vm,
 489                   vlib_node_runtime_t * node, vlib_frame_t * frame)
 490 {
 491   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 492   u32 n_left_from, n_left_to_next, *from, *to_next;
 493   ip_lookup_next_t next;
 494   u32 thread_index = vlib_get_thread_index ();
 495
 496   from = vlib_frame_vector_args (frame);
 497   n_left_from = frame->n_vectors;
 498   next = node->cached_next_index;
 499
 500   if (node->flags & VLIB_NODE_FLAG_TRACE)
 501     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 502
 503   while (n_left_from > 0)
 504     {
 505       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 506
 507
 508       while (n_left_from >= 4 && n_left_to_next >= 2)
 509         {
 510           ip_lookup_next_t next0, next1;
 511           const load_balance_t *lb0, *lb1;
 512           vlib_buffer_t *p0, *p1;
 513           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 514           const ip4_header_t *ip0, *ip1;
 515           const dpo_id_t *dpo0, *dpo1;
 516
 517           /* Prefetch next iteration. */
 518           {
 519             vlib_buffer_t *p2, *p3;
 520
 521             p2 = vlib_get_buffer (vm, from[2]);
 522             p3 = vlib_get_buffer (vm, from[3]);
 523
 524             vlib_prefetch_buffer_header (p2, STORE);
 525             vlib_prefetch_buffer_header (p3, STORE);
 526
 527             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 528             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 529           }
 530
 531           pi0 = to_next[0] = from[0];
 532           pi1 = to_next[1] = from[1];
 533
 534           from += 2;
 535           n_left_from -= 2;
 536           to_next += 2;
 537           n_left_to_next -= 2;
 538
 539           p0 = vlib_get_buffer (vm, pi0);
 540           p1 = vlib_get_buffer (vm, pi1);
 541
 542           ip0 = vlib_buffer_get_current (p0);
 543           ip1 = vlib_buffer_get_current (p1);
 544           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 545           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 546
 547           lb0 = load_balance_get (lbi0);
 548           lb1 = load_balance_get (lbi1);
 549
 550           /*
 551            * this node is for via FIBs we can re-use the hash value from the
 552            * to node if present.
 553            * We don't want to use the same hash value at each level in the recursion
 554            * graph as that would lead to polarisation
 555            */
 556           hc0 = hc1 = 0;
 557
 558           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 559             {
 560               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 561                 {
 562                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 563                     vnet_buffer (p0)->ip.flow_hash >> 1;
 564                 }
 565               else
 566                 {
 567                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 568                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 569                 }
 570               dpo0 = load_balance_get_fwd_bucket
 571                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 572             }
 573           else
 574             {
 575               dpo0 = load_balance_get_bucket_i (lb0, 0);
 576             }
 577           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 578             {
 579               if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash))
 580                 {
 581                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 582                     vnet_buffer (p1)->ip.flow_hash >> 1;
 583                 }
 584               else
 585                 {
 586                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 587                     ip4_compute_flow_hash (ip1, lb1->lb_hash_config);
 588                 }
 589               dpo1 = load_balance_get_fwd_bucket
 590                 (lb1, (hc1 & (lb1->lb_n_buckets_minus_1)));
 591             }
 592           else
 593             {
 594               dpo1 = load_balance_get_bucket_i (lb1, 0);
 595             }
 596
 597           next0 = dpo0->dpoi_next_node;
 598           next1 = dpo1->dpoi_next_node;
 599
 600           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 601           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 602
 603           vlib_increment_combined_counter
 604             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 605           vlib_increment_combined_counter
 606             (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 607
 608           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 609                                            to_next, n_left_to_next,
 610                                            pi0, pi1, next0, next1);
 611         }
 612
 613       while (n_left_from > 0 && n_left_to_next > 0)
 614         {
 615           ip_lookup_next_t next0;
 616           const load_balance_t *lb0;
 617           vlib_buffer_t *p0;
 618           u32 pi0, lbi0, hc0;
 619           const ip4_header_t *ip0;
 620           const dpo_id_t *dpo0;
 621
 622           pi0 = from[0];
 623           to_next[0] = pi0;
 624           from += 1;
 625           to_next += 1;
 626           n_left_to_next -= 1;
 627           n_left_from -= 1;
 628
 629           p0 = vlib_get_buffer (vm, pi0);
 630
 631           ip0 = vlib_buffer_get_current (p0);
 632           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 633
 634           lb0 = load_balance_get (lbi0);
 635
 636           hc0 = 0;
 637           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 638             {
 639               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 640                 {
 641                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 642                     vnet_buffer (p0)->ip.flow_hash >> 1;
 643                 }
 644               else
 645                 {
 646                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 647                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 648                 }
 649               dpo0 = load_balance_get_fwd_bucket
 650                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 651             }
 652           else
 653             {
 654               dpo0 = load_balance_get_bucket_i (lb0, 0);
 655             }
 656
 657           next0 = dpo0->dpoi_next_node;
 658           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 659
 660           vlib_increment_combined_counter
 661             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 662
 663           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 664                                            to_next, n_left_to_next,
 665                                            pi0, next0);
 666         }
 667
 668       vlib_put_next_frame (vm, node, next, n_left_to_next);
 669     }
 670
 671   return frame->n_vectors;
 672 }
 673
 674 VLIB_REGISTER_NODE (ip4_load_balance_node) =
 675 {
 676 .function = ip4_load_balance,.name = "ip4-load-balance",.vector_size =
 677     sizeof (u32),.sibling_of = "ip4-lookup",.format_trace =
 678     format_ip4_lookup_trace,};
 679
 680 VLIB_NODE_FUNCTION_MULTIARCH (ip4_load_balance_node, ip4_load_balance);
 681
 682 /* get first interface address */
 683 ip4_address_t *
 684 ip4_interface_first_address (ip4_main_t * im, u32 sw_if_index,
 685                              ip_interface_address_t ** result_ia)
 686 {
 687   ip_lookup_main_t *lm = &im->lookup_main;
 688   ip_interface_address_t *ia = 0;
 689   ip4_address_t *result = 0;
 690
 691   /* *INDENT-OFF* */
 692   foreach_ip_interface_address
 693     (lm, ia, sw_if_index,
 694      1 /* honor unnumbered */ ,
 695      ({
 696        ip4_address_t * a =
 697          ip_interface_address_get_address (lm, ia);
 698        result = a;
 699        break;
 700      }));
 701   /* *INDENT-OFF* */
 702   if (result_ia)
 703     *result_ia = result ? ia : 0;
 704   return result;
 705 }
 706
 707 static void
 708 ip4_add_interface_routes (u32 sw_if_index,
 709                           ip4_main_t * im, u32 fib_index,
 710                           ip_interface_address_t * a)
 711 {
 712   ip_lookup_main_t *lm = &im->lookup_main;
 713   ip4_address_t *address = ip_interface_address_get_address (lm, a);
 714   fib_prefix_t pfx = {
 715     .fp_len = a->address_length,
 716     .fp_proto = FIB_PROTOCOL_IP4,
 717     .fp_addr.ip4 = *address,
 718   };
 719
 720   if (pfx.fp_len <= 30)
 721     {
 722       /* a /30 or shorter - add a glean for the network address */
 723       fib_table_entry_update_one_path (fib_index, &pfx,
 724                                        FIB_SOURCE_INTERFACE,
 725                                        (FIB_ENTRY_FLAG_CONNECTED |
 726                                         FIB_ENTRY_FLAG_ATTACHED),
 727                                        FIB_PROTOCOL_IP4,
 728                                        /* No next-hop address */
 729                                        NULL,
 730                                        sw_if_index,
 731                                        // invalid FIB index
 732                                        ~0,
 733                                        1,
 734                                        // no out-label stack
 735                                        NULL,
 736                                        FIB_ROUTE_PATH_FLAG_NONE);
 737
 738       /* Add the two broadcast addresses as drop */
 739       fib_prefix_t net_pfx = {
 740         .fp_len = 32,
 741         .fp_proto = FIB_PROTOCOL_IP4,
 742         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 743       };
 744       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 745         fib_table_entry_special_add(fib_index,
 746                                     &net_pfx,
 747                                     FIB_SOURCE_INTERFACE,
 748                                     (FIB_ENTRY_FLAG_DROP |
 749                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 750       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 751       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 752         fib_table_entry_special_add(fib_index,
 753                                     &net_pfx,
 754                                     FIB_SOURCE_INTERFACE,
 755                                     (FIB_ENTRY_FLAG_DROP |
 756                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 757     }
 758   else if (pfx.fp_len == 31)
 759     {
 760       u32 mask = clib_host_to_net_u32(1);
 761       fib_prefix_t net_pfx = pfx;
 762
 763       net_pfx.fp_len = 32;
 764       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 765
 766       /* a /31 - add the other end as an attached host */
 767       fib_table_entry_update_one_path (fib_index, &net_pfx,
 768                                        FIB_SOURCE_INTERFACE,
 769                                        (FIB_ENTRY_FLAG_ATTACHED),
 770                                        FIB_PROTOCOL_IP4,
 771                                        &net_pfx.fp_addr,
 772                                        sw_if_index,
 773                                        // invalid FIB index
 774                                        ~0,
 775                                        1,
 776                                        NULL,
 777                                        FIB_ROUTE_PATH_FLAG_NONE);
 778     }
 779   pfx.fp_len = 32;
 780
 781   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 782     {
 783       u32 classify_table_index =
 784         lm->classify_table_index_by_sw_if_index[sw_if_index];
 785       if (classify_table_index != (u32) ~ 0)
 786         {
 787           dpo_id_t dpo = DPO_INVALID;
 788
 789           dpo_set (&dpo,
 790                    DPO_CLASSIFY,
 791                    DPO_PROTO_IP4,
 792                    classify_dpo_create (DPO_PROTO_IP4, classify_table_index));
 793
 794           fib_table_entry_special_dpo_add (fib_index,
 795                                            &pfx,
 796                                            FIB_SOURCE_CLASSIFY,
 797                                            FIB_ENTRY_FLAG_NONE, &dpo);
 798           dpo_reset (&dpo);
 799         }
 800     }
 801
 802   fib_table_entry_update_one_path (fib_index, &pfx,
 803                                    FIB_SOURCE_INTERFACE,
 804                                    (FIB_ENTRY_FLAG_CONNECTED |
 805                                     FIB_ENTRY_FLAG_LOCAL),
 806                                    FIB_PROTOCOL_IP4,
 807                                    &pfx.fp_addr,
 808                                    sw_if_index,
 809                                    // invalid FIB index
 810                                    ~0,
 811                                    1, NULL,
 812                                    FIB_ROUTE_PATH_FLAG_NONE);
 813 }
 814
 815 static void
 816 ip4_del_interface_routes (ip4_main_t * im,
 817                           u32 fib_index,
 818                           ip4_address_t * address, u32 address_length)
 819 {
 820   fib_prefix_t pfx = {
 821     .fp_len = address_length,
 822     .fp_proto = FIB_PROTOCOL_IP4,
 823     .fp_addr.ip4 = *address,
 824   };
 825
 826   if (pfx.fp_len <= 30)
 827     {
 828       fib_prefix_t net_pfx = {
 829         .fp_len = 32,
 830         .fp_proto = FIB_PROTOCOL_IP4,
 831         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 832       };
 833       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 834         fib_table_entry_special_remove(fib_index,
 835                                        &net_pfx,
 836                                        FIB_SOURCE_INTERFACE);
 837       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 838       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 839         fib_table_entry_special_remove(fib_index,
 840                                        &net_pfx,
 841                                        FIB_SOURCE_INTERFACE);
 842       fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 843     }
 844     else if (pfx.fp_len == 31)
 845     {
 846       u32 mask = clib_host_to_net_u32(1);
 847       fib_prefix_t net_pfx = pfx;
 848
 849       net_pfx.fp_len = 32;
 850       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 851
 852       fib_table_entry_delete (fib_index, &net_pfx, FIB_SOURCE_INTERFACE);
 853     }
 854
 855   pfx.fp_len = 32;
 856   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 857 }
 858
 859 void
 860 ip4_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 861 {
 862   ip4_main_t *im = &ip4_main;
 863
 864   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 865
 866   /*
 867    * enable/disable only on the 1<->0 transition
 868    */
 869   if (is_enable)
 870     {
 871       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 872         return;
 873     }
 874   else
 875     {
 876       ASSERT (im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 877       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 878         return;
 879     }
 880   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
 881                                !is_enable, 0, 0);
 882
 883
 884   vnet_feature_enable_disable ("ip4-multicast", "ip4-drop",
 885                                sw_if_index, !is_enable, 0, 0);
 886 }
 887
 888 static clib_error_t *
 889 ip4_add_del_interface_address_internal (vlib_main_t * vm,
 890                                         u32 sw_if_index,
 891                                         ip4_address_t * address,
 892                                         u32 address_length, u32 is_del)
 893 {
 894   vnet_main_t *vnm = vnet_get_main ();
 895   ip4_main_t *im = &ip4_main;
 896   ip_lookup_main_t *lm = &im->lookup_main;
 897   clib_error_t *error = 0;
 898   u32 if_address_index, elts_before;
 899   ip4_address_fib_t ip4_af, *addr_fib = 0;
 900
 901   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 902   ip4_addr_fib_init (&ip4_af, address,
 903                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 904   vec_add1 (addr_fib, ip4_af);
 905
 906   /* FIXME-LATER
 907    * there is no support for adj-fib handling in the presence of overlapping
 908    * subnets on interfaces. Easy fix - disallow overlapping subnets, like
 909    * most routers do.
 910    */
 911   /* *INDENT-OFF* */
 912   if (!is_del)
 913     {
 914       /* When adding an address check that it does not conflict
 915          with an existing address. */
 916       ip_interface_address_t *ia;
 917       foreach_ip_interface_address
 918         (&im->lookup_main, ia, sw_if_index,
 919          0 /* honor unnumbered */ ,
 920          ({
 921            ip4_address_t * x =
 922              ip_interface_address_get_address
 923              (&im->lookup_main, ia);
 924            if (ip4_destination_matches_route
 925                (im, address, x, ia->address_length) ||
 926                ip4_destination_matches_route (im,
 927                                               x,
 928                                               address,
 929                                               address_length))
 930              return
 931                clib_error_create
 932                ("failed to add %U which conflicts with %U for interface %U",
 933                 format_ip4_address_and_length, address,
 934                 address_length,
 935                 format_ip4_address_and_length, x,
 936                 ia->address_length,
 937                 format_vnet_sw_if_index_name, vnm,
 938                 sw_if_index);
 939          }));
 940     }
 941   /* *INDENT-ON* */
 942
 943   elts_before = pool_elts (lm->if_address_pool);
 944
 945   error = ip_interface_address_add_del
 946     (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 947   if (error)
 948     goto done;
 949
 950   ip4_sw_interface_enable_disable (sw_if_index, !is_del);
 951
 952   if (is_del)
 953     ip4_del_interface_routes (im, ip4_af.fib_index, address, address_length);
 954   else
 955     ip4_add_interface_routes (sw_if_index,
 956                               im, ip4_af.fib_index,
 957                               pool_elt_at_index
 958                               (lm->if_address_pool, if_address_index));
 959
 960   /* If pool did not grow/shrink: add duplicate address. */
 961   if (elts_before != pool_elts (lm->if_address_pool))
 962     {
 963       ip4_add_del_interface_address_callback_t *cb;
 964       vec_foreach (cb, im->add_del_interface_address_callbacks)
 965         cb->function (im, cb->function_opaque, sw_if_index,
 966                       address, address_length, if_address_index, is_del);
 967     }
 968
 969 done:
 970   vec_free (addr_fib);
 971   return error;
 972 }
 973
 974 clib_error_t *
 975 ip4_add_del_interface_address (vlib_main_t * vm,
 976                                u32 sw_if_index,
 977                                ip4_address_t * address,
 978                                u32 address_length, u32 is_del)
 979 {
 980   return ip4_add_del_interface_address_internal
 981     (vm, sw_if_index, address, address_length, is_del);
 982 }
 983
 984 /* Built-in ip4 unicast rx feature path definition */
 985 /* *INDENT-OFF* */
 986 VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
 987 {
 988   .arc_name = "ip4-unicast",
 989   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 990   .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
 991 };
 992
 993 VNET_FEATURE_INIT (ip4_flow_classify, static) =
 994 {
 995   .arc_name = "ip4-unicast",
 996   .node_name = "ip4-flow-classify",
 997   .runs_before = VNET_FEATURES ("ip4-inacl"),
 998 };
 999
1000 VNET_FEATURE_INIT (ip4_inacl, static) =
1001 {
1002   .arc_name = "ip4-unicast",
1003   .node_name = "ip4-inacl",
1004   .runs_before = VNET_FEATURES ("ip4-source-check-via-rx"),
1005 };
1006
1007 VNET_FEATURE_INIT (ip4_source_check_1, static) =
1008 {
1009   .arc_name = "ip4-unicast",
1010   .node_name = "ip4-source-check-via-rx",
1011   .runs_before = VNET_FEATURES ("ip4-source-check-via-any"),
1012 };
1013
1014 VNET_FEATURE_INIT (ip4_source_check_2, static) =
1015 {
1016   .arc_name = "ip4-unicast",
1017   .node_name = "ip4-source-check-via-any",
1018   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
1019 };
1020
1021 VNET_FEATURE_INIT (ip4_source_and_port_range_check_rx, static) =
1022 {
1023   .arc_name = "ip4-unicast",
1024   .node_name = "ip4-source-and-port-range-check-rx",
1025   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
1026 };
1027
1028 VNET_FEATURE_INIT (ip4_policer_classify, static) =
1029 {
1030   .arc_name = "ip4-unicast",
1031   .node_name = "ip4-policer-classify",
1032   .runs_before = VNET_FEATURES ("ipsec-input-ip4"),
1033 };
1034
1035 VNET_FEATURE_INIT (ip4_ipsec, static) =
1036 {
1037   .arc_name = "ip4-unicast",
1038   .node_name = "ipsec-input-ip4",
1039   .runs_before = VNET_FEATURES ("vpath-input-ip4"),
1040 };
1041
1042 VNET_FEATURE_INIT (ip4_vpath, static) =
1043 {
1044   .arc_name = "ip4-unicast",
1045   .node_name = "vpath-input-ip4",
1046   .runs_before = VNET_FEATURES ("ip4-vxlan-bypass"),
1047 };
1048
1049 VNET_FEATURE_INIT (ip4_vxlan_bypass, static) =
1050 {
1051   .arc_name = "ip4-unicast",
1052   .node_name = "ip4-vxlan-bypass",
1053   .runs_before = VNET_FEATURES ("ip4-lookup"),
1054 };
1055
1056 VNET_FEATURE_INIT (ip4_drop, static) =
1057 {
1058   .arc_name = "ip4-unicast",
1059   .node_name = "ip4-drop",
1060   .runs_before = VNET_FEATURES ("ip4-lookup"),
1061 };
1062
1063 VNET_FEATURE_INIT (ip4_lookup, static) =
1064 {
1065   .arc_name = "ip4-unicast",
1066   .node_name = "ip4-lookup",
1067   .runs_before = 0,     /* not before any other features */
1068 };
1069
1070 /* Built-in ip4 multicast rx feature path definition */
1071 VNET_FEATURE_ARC_INIT (ip4_multicast, static) =
1072 {
1073   .arc_name = "ip4-multicast",
1074   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
1075   .arc_index_ptr = &ip4_main.lookup_main.mcast_feature_arc_index,
1076 };
1077
1078 VNET_FEATURE_INIT (ip4_vpath_mc, static) =
1079 {
1080   .arc_name = "ip4-multicast",
1081   .node_name = "vpath-input-ip4",
1082   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
1083 };
1084
1085 VNET_FEATURE_INIT (ip4_mc_drop, static) =
1086 {
1087   .arc_name = "ip4-multicast",
1088   .node_name = "ip4-drop",
1089   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
1090 };
1091
1092 VNET_FEATURE_INIT (ip4_lookup_mc, static) =
1093 {
1094   .arc_name = "ip4-multicast",
1095   .node_name = "ip4-mfib-forward-lookup",
1096   .runs_before = 0,     /* last feature */
1097 };
1098
1099 /* Source and port-range check ip4 tx feature path definition */
1100 VNET_FEATURE_ARC_INIT (ip4_output, static) =
1101 {
1102   .arc_name = "ip4-output",
1103   .start_nodes = VNET_FEATURES ("ip4-rewrite", "ip4-midchain"),
1104   .arc_index_ptr = &ip4_main.lookup_main.output_feature_arc_index,
1105 };
1106
1107 VNET_FEATURE_INIT (ip4_source_and_port_range_check_tx, static) =
1108 {
1109   .arc_name = "ip4-output",
1110   .node_name = "ip4-source-and-port-range-check-tx",
1111   .runs_before = VNET_FEATURES ("ipsec-output-ip4"),
1112 };
1113
1114 VNET_FEATURE_INIT (ip4_ipsec_output, static) =
1115 {
1116   .arc_name = "ip4-output",
1117   .node_name = "ipsec-output-ip4",
1118   .runs_before = VNET_FEATURES ("interface-output"),
1119 };
1120
1121 /* Built-in ip4 tx feature path definition */
1122 VNET_FEATURE_INIT (ip4_interface_output, static) =
1123 {
1124   .arc_name = "ip4-output",
1125   .node_name = "interface-output",
1126   .runs_before = 0,     /* not before any other features */
1127 };
1128 /* *INDENT-ON* */
1129
1130 static clib_error_t *
1131 ip4_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
1132 {
1133   ip4_main_t *im = &ip4_main;
1134
1135   /* Fill in lookup tables with default table (0). */
1136   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1137   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
1138
1139   if (!is_add)
1140     {
1141       ip4_main_t *im4 = &ip4_main;
1142       ip_lookup_main_t *lm4 = &im4->lookup_main;
1143       ip_interface_address_t *ia = 0;
1144       ip4_address_t *address;
1145       vlib_main_t *vm = vlib_get_main ();
1146
1147       /* *INDENT-OFF* */
1148       foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* honor unnumbered */,
1149       ({
1150         address = ip_interface_address_get_address (lm4, ia);
1151         ip4_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
1152       }));
1153       /* *INDENT-ON* */
1154     }
1155
1156   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
1157                                is_add, 0, 0);
1158
1159   vnet_feature_enable_disable ("ip4-multicast", "ip4-drop", sw_if_index,
1160                                is_add, 0, 0);
1161
1162   return /* no error */ 0;
1163 }
1164
1165 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
1166
1167 /* Global IP4 main. */
1168 ip4_main_t ip4_main;
1169
1170 clib_error_t *
1171 ip4_lookup_init (vlib_main_t * vm)
1172 {
1173   ip4_main_t *im = &ip4_main;
1174   clib_error_t *error;
1175   uword i;
1176
1177   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
1178     return error;
1179
1180   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
1181     {
1182       u32 m;
1183
1184       if (i < 32)
1185         m = pow2_mask (i) << (32 - i);
1186       else
1187         m = ~0;
1188       im->fib_masks[i] = clib_host_to_net_u32 (m);
1189     }
1190
1191   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
1192
1193   /* Create FIB with index 0 and table id of 0. */
1194   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0);
1195   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0);
1196
1197   {
1198     pg_node_t *pn;
1199     pn = pg_get_node (ip4_lookup_node.index);
1200     pn->unformat_edit = unformat_pg_ip4_header;
1201   }
1202
1203   {
1204     ethernet_arp_header_t h;
1205
1206     memset (&h, 0, sizeof (h));
1207
1208     /* Set target ethernet address to all zeros. */
1209     memset (h.ip4_over_ethernet[1].ethernet, 0,
1210             sizeof (h.ip4_over_ethernet[1].ethernet));
1211
1212 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
1213 #define _8(f,v) h.f = v;
1214     _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
1215     _16 (l3_type, ETHERNET_TYPE_IP4);
1216     _8 (n_l2_address_bytes, 6);
1217     _8 (n_l3_address_bytes, 4);
1218     _16 (opcode, ETHERNET_ARP_OPCODE_request);
1219 #undef _16
1220 #undef _8
1221
1222     vlib_packet_template_init (vm, &im->ip4_arp_request_packet_template,
1223                                /* data */ &h,
1224                                sizeof (h),
1225                                /* alloc chunk size */ 8,
1226                                "ip4 arp");
1227   }
1228
1229   return error;
1230 }
1231
1232 VLIB_INIT_FUNCTION (ip4_lookup_init);
1233
1234 typedef struct
1235 {
1236   /* Adjacency taken. */
1237   u32 dpo_index;
1238   u32 flow_hash;
1239   u32 fib_index;
1240
1241   /* Packet data, possibly *after* rewrite. */
1242   u8 packet_data[64 - 1 * sizeof (u32)];
1243 }
1244 ip4_forward_next_trace_t;
1245
1246 u8 *
1247 format_ip4_forward_next_trace (u8 * s, va_list * args)
1248 {
1249   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1250   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1251   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1252   uword indent = format_get_indent (s);
1253   s = format (s, "%U%U",
1254               format_white_space, indent,
1255               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1256   return s;
1257 }
1258
1259 static u8 *
1260 format_ip4_lookup_trace (u8 * s, va_list * args)
1261 {
1262   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1263   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1264   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1265   uword indent = format_get_indent (s);
1266
1267   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
1268               t->fib_index, t->dpo_index, t->flow_hash);
1269   s = format (s, "\n%U%U",
1270               format_white_space, indent,
1271               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1272   return s;
1273 }
1274
1275 static u8 *
1276 format_ip4_rewrite_trace (u8 * s, va_list * args)
1277 {
1278   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1279   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1280   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1281   uword indent = format_get_indent (s);
1282
1283   s = format (s, "tx_sw_if_index %d dpo-idx %d : %U flow hash: 0x%08x",
1284               t->fib_index, t->dpo_index, format_ip_adjacency,
1285               t->dpo_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
1286   s = format (s, "\n%U%U",
1287               format_white_space, indent,
1288               format_ip_adjacency_packet_data,
1289               t->dpo_index, t->packet_data, sizeof (t->packet_data));
1290   return s;
1291 }
1292
1293 /* Common trace function for all ip4-forward next nodes. */
1294 void
1295 ip4_forward_next_trace (vlib_main_t * vm,
1296                         vlib_node_runtime_t * node,
1297                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
1298 {
1299   u32 *from, n_left;
1300   ip4_main_t *im = &ip4_main;
1301
1302   n_left = frame->n_vectors;
1303   from = vlib_frame_vector_args (frame);
1304
1305   while (n_left >= 4)
1306     {
1307       u32 bi0, bi1;
1308       vlib_buffer_t *b0, *b1;
1309       ip4_forward_next_trace_t *t0, *t1;
1310
1311       /* Prefetch next iteration. */
1312       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1313       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1314
1315       bi0 = from[0];
1316       bi1 = from[1];
1317
1318       b0 = vlib_get_buffer (vm, bi0);
1319       b1 = vlib_get_buffer (vm, bi1);
1320
1321       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1322         {
1323           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1324           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1325           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1326           t0->fib_index =
1327             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1328              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1329             vec_elt (im->fib_index_by_sw_if_index,
1330                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1331
1332           clib_memcpy (t0->packet_data,
1333                        vlib_buffer_get_current (b0),
1334                        sizeof (t0->packet_data));
1335         }
1336       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1337         {
1338           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1339           t1->dpo_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1340           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1341           t1->fib_index =
1342             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
1343              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
1344             vec_elt (im->fib_index_by_sw_if_index,
1345                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
1346           clib_memcpy (t1->packet_data, vlib_buffer_get_current (b1),
1347                        sizeof (t1->packet_data));
1348         }
1349       from += 2;
1350       n_left -= 2;
1351     }
1352
1353   while (n_left >= 1)
1354     {
1355       u32 bi0;
1356       vlib_buffer_t *b0;
1357       ip4_forward_next_trace_t *t0;
1358
1359       bi0 = from[0];
1360
1361       b0 = vlib_get_buffer (vm, bi0);
1362
1363       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1364         {
1365           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1366           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1367           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1368           t0->fib_index =
1369             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1370              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1371             vec_elt (im->fib_index_by_sw_if_index,
1372                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1373           clib_memcpy (t0->packet_data, vlib_buffer_get_current (b0),
1374                        sizeof (t0->packet_data));
1375         }
1376       from += 1;
1377       n_left -= 1;
1378     }
1379 }
1380
1381 static uword
1382 ip4_drop_or_punt (vlib_main_t * vm,
1383                   vlib_node_runtime_t * node,
1384                   vlib_frame_t * frame, ip4_error_t error_code)
1385 {
1386   u32 *buffers = vlib_frame_vector_args (frame);
1387   uword n_packets = frame->n_vectors;
1388
1389   vlib_error_drop_buffers (vm, node, buffers,
1390                            /* stride */ 1,
1391                            n_packets,
1392                            /* next */ 0,
1393                            ip4_input_node.index, error_code);
1394
1395   if (node->flags & VLIB_NODE_FLAG_TRACE)
1396     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1397
1398   return n_packets;
1399 }
1400
1401 static uword
1402 ip4_drop (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1403 {
1404   return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_DROP);
1405 }
1406
1407 static uword
1408 ip4_punt (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1409 {
1410   return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_PUNT);
1411 }
1412
1413 /* *INDENT-OFF* */
1414 VLIB_REGISTER_NODE (ip4_drop_node, static) =
1415 {
1416   .function = ip4_drop,
1417   .name = "ip4-drop",
1418   .vector_size = sizeof (u32),
1419   .format_trace = format_ip4_forward_next_trace,
1420   .n_next_nodes = 1,
1421   .next_nodes = {
1422     [0] = "error-drop",
1423   },
1424 };
1425
1426 VLIB_NODE_FUNCTION_MULTIARCH (ip4_drop_node, ip4_drop);
1427
1428 VLIB_REGISTER_NODE (ip4_punt_node, static) =
1429 {
1430   .function = ip4_punt,
1431   .name = "ip4-punt",
1432   .vector_size = sizeof (u32),
1433   .format_trace = format_ip4_forward_next_trace,
1434   .n_next_nodes = 1,
1435   .next_nodes = {
1436     [0] = "error-punt",
1437   },
1438 };
1439
1440 VLIB_NODE_FUNCTION_MULTIARCH (ip4_punt_node, ip4_punt);
1441 /* *INDENT-ON */
1442
1443 /* Compute TCP/UDP/ICMP4 checksum in software. */
1444 u16
1445 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1446                               ip4_header_t * ip0)
1447 {
1448   ip_csum_t sum0;
1449   u32 ip_header_length, payload_length_host_byte_order;
1450   u32 n_this_buffer, n_bytes_left;
1451   u16 sum16;
1452   void *data_this_buffer;
1453
1454   /* Initialize checksum with ip header. */
1455   ip_header_length = ip4_header_bytes (ip0);
1456   payload_length_host_byte_order =
1457     clib_net_to_host_u16 (ip0->length) - ip_header_length;
1458   sum0 =
1459     clib_host_to_net_u32 (payload_length_host_byte_order +
1460                           (ip0->protocol << 16));
1461
1462   if (BITS (uword) == 32)
1463     {
1464       sum0 =
1465         ip_csum_with_carry (sum0,
1466                             clib_mem_unaligned (&ip0->src_address, u32));
1467       sum0 =
1468         ip_csum_with_carry (sum0,
1469                             clib_mem_unaligned (&ip0->dst_address, u32));
1470     }
1471   else
1472     sum0 =
1473       ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1474
1475   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1476   data_this_buffer = (void *) ip0 + ip_header_length;
1477   if (n_this_buffer + ip_header_length > p0->current_length)
1478     n_this_buffer =
1479       p0->current_length >
1480       ip_header_length ? p0->current_length - ip_header_length : 0;
1481   while (1)
1482     {
1483       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1484       n_bytes_left -= n_this_buffer;
1485       if (n_bytes_left == 0)
1486         break;
1487
1488       ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1489       p0 = vlib_get_buffer (vm, p0->next_buffer);
1490       data_this_buffer = vlib_buffer_get_current (p0);
1491       n_this_buffer = p0->current_length;
1492     }
1493
1494   sum16 = ~ip_csum_fold (sum0);
1495
1496   return sum16;
1497 }
1498
1499 u32
1500 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1501 {
1502   ip4_header_t *ip0 = vlib_buffer_get_current (p0);
1503   udp_header_t *udp0;
1504   u16 sum16;
1505
1506   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1507           || ip0->protocol == IP_PROTOCOL_UDP);
1508
1509   udp0 = (void *) (ip0 + 1);
1510   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1511     {
1512       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1513                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1514       return p0->flags;
1515     }
1516
1517   sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1518
1519   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1520                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1521
1522   return p0->flags;
1523 }
1524
1525 /* *INDENT-OFF* */
1526 VNET_FEATURE_ARC_INIT (ip4_local) =
1527 {
1528   .arc_name  = "ip4-local",
1529   .start_nodes = VNET_FEATURES ("ip4-local"),
1530 };
1531 /* *INDENT-ON* */
1532
1533 static inline uword
1534 ip4_local_inline (vlib_main_t * vm,
1535                   vlib_node_runtime_t * node,
1536                   vlib_frame_t * frame, int head_of_feature_arc)
1537 {
1538   ip4_main_t *im = &ip4_main;
1539   ip_lookup_main_t *lm = &im->lookup_main;
1540   ip_local_next_t next_index;
1541   u32 *from, *to_next, n_left_from, n_left_to_next;
1542   vlib_node_runtime_t *error_node =
1543     vlib_node_get_runtime (vm, ip4_input_node.index);
1544   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1545
1546   from = vlib_frame_vector_args (frame);
1547   n_left_from = frame->n_vectors;
1548   next_index = node->cached_next_index;
1549
1550   if (node->flags & VLIB_NODE_FLAG_TRACE)
1551     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1552
1553   while (n_left_from > 0)
1554     {
1555       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1556
1557       while (n_left_from >= 4 && n_left_to_next >= 2)
1558         {
1559           vlib_buffer_t *p0, *p1;
1560           ip4_header_t *ip0, *ip1;
1561           udp_header_t *udp0, *udp1;
1562           ip4_fib_mtrie_t *mtrie0, *mtrie1;
1563           ip4_fib_mtrie_leaf_t leaf0, leaf1;
1564           const dpo_id_t *dpo0, *dpo1;
1565           const load_balance_t *lb0, *lb1;
1566           u32 pi0, ip_len0, udp_len0, flags0, next0, fib_index0, lbi0;
1567           u32 pi1, ip_len1, udp_len1, flags1, next1, fib_index1, lbi1;
1568           i32 len_diff0, len_diff1;
1569           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1570           u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1571           u32 sw_if_index0, sw_if_index1;
1572
1573           pi0 = to_next[0] = from[0];
1574           pi1 = to_next[1] = from[1];
1575           from += 2;
1576           n_left_from -= 2;
1577           to_next += 2;
1578           n_left_to_next -= 2;
1579
1580           next0 = next1 = IP_LOCAL_NEXT_DROP;
1581
1582           p0 = vlib_get_buffer (vm, pi0);
1583           p1 = vlib_get_buffer (vm, pi1);
1584
1585           ip0 = vlib_buffer_get_current (p0);
1586           ip1 = vlib_buffer_get_current (p1);
1587
1588           vnet_buffer (p0)->ip.start_of_ip_header = p0->current_data;
1589           vnet_buffer (p1)->ip.start_of_ip_header = p1->current_data;
1590
1591           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1592           sw_if_index1 = vnet_buffer (p1)->sw_if_index[VLIB_RX];
1593
1594           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1595           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index1);
1596
1597           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1598           fib_index0 =
1599             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1600              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1601
1602           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index1);
1603           fib_index1 =
1604             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
1605              (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
1606
1607           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1608           mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
1609
1610           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1611           leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, &ip1->src_address);
1612
1613           /* Treat IP frag packets as "experimental" protocol for now
1614              until support of IP frag reassembly is implemented */
1615           proto0 = ip4_is_fragment (ip0) ? 0xfe : ip0->protocol;
1616           proto1 = ip4_is_fragment (ip1) ? 0xfe : ip1->protocol;
1617
1618           if (head_of_feature_arc == 0)
1619             {
1620               error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1621               goto skip_checks;
1622             }
1623
1624           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1625           is_udp1 = proto1 == IP_PROTOCOL_UDP;
1626           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1627           is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1628
1629           flags0 = p0->flags;
1630           flags1 = p1->flags;
1631
1632           good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1633           good_tcp_udp1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1634
1635           udp0 = ip4_next_header (ip0);
1636           udp1 = ip4_next_header (ip1);
1637
1638           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1639           good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1640           good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1641
1642           /* Verify UDP length. */
1643           ip_len0 = clib_net_to_host_u16 (ip0->length);
1644           ip_len1 = clib_net_to_host_u16 (ip1->length);
1645           udp_len0 = clib_net_to_host_u16 (udp0->length);
1646           udp_len1 = clib_net_to_host_u16 (udp1->length);
1647
1648           len_diff0 = ip_len0 - udp_len0;
1649           len_diff1 = ip_len1 - udp_len1;
1650
1651           len_diff0 = is_udp0 ? len_diff0 : 0;
1652           len_diff1 = is_udp1 ? len_diff1 : 0;
1653
1654           if (PREDICT_FALSE (!(is_tcp_udp0 & is_tcp_udp1
1655                                & good_tcp_udp0 & good_tcp_udp1)))
1656             {
1657               if (is_tcp_udp0)
1658                 {
1659                   if (is_tcp_udp0
1660                       && !(flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1661                     flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
1662                   good_tcp_udp0 =
1663                     (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1664                   good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1665                 }
1666               if (is_tcp_udp1)
1667                 {
1668                   if (is_tcp_udp1
1669                       && !(flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1670                     flags1 = ip4_tcp_udp_validate_checksum (vm, p1);
1671                   good_tcp_udp1 =
1672                     (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1673                   good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1674                 }
1675             }
1676
1677           good_tcp_udp0 &= len_diff0 >= 0;
1678           good_tcp_udp1 &= len_diff1 >= 0;
1679
1680           leaf0 =
1681             ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1682           leaf1 =
1683             ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 2);
1684
1685           error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1686
1687           error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
1688           error1 = len_diff1 < 0 ? IP4_ERROR_UDP_LENGTH : error1;
1689
1690           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1691           error0 = (is_tcp_udp0 && !good_tcp_udp0
1692                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1693           error1 = (is_tcp_udp1 && !good_tcp_udp1
1694                     ? IP4_ERROR_TCP_CHECKSUM + is_udp1 : error1);
1695
1696           leaf0 =
1697             ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1698           leaf1 =
1699             ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 3);
1700
1701           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0 =
1702             ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1703           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1704
1705           vnet_buffer (p1)->ip.adj_index[VLIB_RX] = lbi1 =
1706             ip4_fib_mtrie_leaf_get_adj_index (leaf1);
1707           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = lbi1;
1708
1709           lb0 = load_balance_get (lbi0);
1710           lb1 = load_balance_get (lbi1);
1711           dpo0 = load_balance_get_bucket_i (lb0, 0);
1712           dpo1 = load_balance_get_bucket_i (lb1, 0);
1713
1714           /*
1715            * Must have a route to source otherwise we drop the packet.
1716            * ip4 broadcasts are accepted, e.g. to make dhcp client work
1717            *
1718            * The checks are:
1719            *  - the source is a recieve => it's from us => bogus, do this
1720            *    first since it sets a different error code.
1721            *  - uRPF check for any route to source - accept if passes.
1722            *  - allow packets destined to the broadcast address from unknown sources
1723            */
1724           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1725                      dpo0->dpoi_type == DPO_RECEIVE) ?
1726                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1727           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1728                      !fib_urpf_check_size (lb0->lb_urpf) &&
1729                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1730                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1731           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1732                      dpo1->dpoi_type == DPO_RECEIVE) ?
1733                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error1);
1734           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1735                      !fib_urpf_check_size (lb1->lb_urpf) &&
1736                      ip1->dst_address.as_u32 != 0xFFFFFFFF)
1737                     ? IP4_ERROR_SRC_LOOKUP_MISS : error1);
1738
1739         skip_checks:
1740
1741           next0 = lm->local_next_by_ip_protocol[proto0];
1742           next1 = lm->local_next_by_ip_protocol[proto1];
1743
1744           next0 =
1745             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1746           next1 =
1747             error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1748
1749           p0->error = error0 ? error_node->errors[error0] : 0;
1750           p1->error = error1 ? error_node->errors[error1] : 0;
1751
1752           if (head_of_feature_arc)
1753             {
1754               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1755                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1756               if (PREDICT_TRUE (error1 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1757                 vnet_feature_arc_start (arc_index, sw_if_index1, &next1, p1);
1758             }
1759
1760           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1761                                            n_left_to_next, pi0, pi1,
1762                                            next0, next1);
1763         }
1764
1765       while (n_left_from > 0 && n_left_to_next > 0)
1766         {
1767           vlib_buffer_t *p0;
1768           ip4_header_t *ip0;
1769           udp_header_t *udp0;
1770           ip4_fib_mtrie_t *mtrie0;
1771           ip4_fib_mtrie_leaf_t leaf0;
1772           u32 pi0, next0, ip_len0, udp_len0, flags0, fib_index0, lbi0;
1773           i32 len_diff0;
1774           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1775           load_balance_t *lb0;
1776           const dpo_id_t *dpo0;
1777           u32 sw_if_index0;
1778
1779           pi0 = to_next[0] = from[0];
1780           from += 1;
1781           n_left_from -= 1;
1782           to_next += 1;
1783           n_left_to_next -= 1;
1784
1785           next0 = IP_LOCAL_NEXT_DROP;
1786
1787           p0 = vlib_get_buffer (vm, pi0);
1788
1789           ip0 = vlib_buffer_get_current (p0);
1790
1791           vnet_buffer (p0)->ip.start_of_ip_header = p0->current_data;
1792
1793           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1794
1795           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1796
1797           fib_index0 =
1798             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1799              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1800
1801           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1802
1803           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1804
1805           /* Treat IP frag packets as "experimental" protocol for now
1806              until support of IP frag reassembly is implemented */
1807           proto0 = ip4_is_fragment (ip0) ? 0xfe : ip0->protocol;
1808
1809           if (head_of_feature_arc == 0)
1810             {
1811               error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
1812               goto skip_check;
1813             }
1814
1815           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1816           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1817
1818           flags0 = p0->flags;
1819
1820           good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1821
1822           udp0 = ip4_next_header (ip0);
1823
1824           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1825           good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1826
1827           /* Verify UDP length. */
1828           ip_len0 = clib_net_to_host_u16 (ip0->length);
1829           udp_len0 = clib_net_to_host_u16 (udp0->length);
1830
1831           len_diff0 = ip_len0 - udp_len0;
1832
1833           len_diff0 = is_udp0 ? len_diff0 : 0;
1834
1835           if (PREDICT_FALSE (!(is_tcp_udp0 & good_tcp_udp0)))
1836             {
1837               if (is_tcp_udp0)
1838                 {
1839                   if (is_tcp_udp0
1840                       && !(flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1841                     flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
1842                   good_tcp_udp0 =
1843                     (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1844                   good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1845                 }
1846             }
1847
1848           good_tcp_udp0 &= len_diff0 >= 0;
1849
1850           leaf0 =
1851             ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1852
1853           error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
1854
1855           error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
1856
1857           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1858           error0 = (is_tcp_udp0 && !good_tcp_udp0
1859                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1860
1861           leaf0 =
1862             ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1863
1864           lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1865           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1866
1867           lb0 = load_balance_get (lbi0);
1868           dpo0 = load_balance_get_bucket_i (lb0, 0);
1869
1870           vnet_buffer (p0)->ip.adj_index[VLIB_TX] =
1871             vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0;
1872
1873           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1874                      dpo0->dpoi_type == DPO_RECEIVE) ?
1875                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1876           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1877                      !fib_urpf_check_size (lb0->lb_urpf) &&
1878                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1879                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1880
1881         skip_check:
1882
1883           next0 = lm->local_next_by_ip_protocol[proto0];
1884
1885           next0 =
1886             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1887
1888           p0->error = error0 ? error_node->errors[error0] : 0;
1889
1890           if (head_of_feature_arc)
1891             {
1892               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1893                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1894             }
1895
1896           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1897                                            n_left_to_next, pi0, next0);
1898
1899         }
1900
1901       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1902     }
1903
1904   return frame->n_vectors;
1905 }
1906
1907 static uword
1908 ip4_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1909 {
1910   return ip4_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1911 }
1912
1913 /* *INDENT-OFF* */
1914 VLIB_REGISTER_NODE (ip4_local_node) =
1915 {
1916   .function = ip4_local,
1917   .name = "ip4-local",
1918   .vector_size = sizeof (u32),
1919   .format_trace = format_ip4_forward_next_trace,
1920   .n_next_nodes = IP_LOCAL_N_NEXT,
1921   .next_nodes =
1922   {
1923     [IP_LOCAL_NEXT_DROP] = "error-drop",
1924     [IP_LOCAL_NEXT_PUNT] = "error-punt",
1925     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
1926     [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",},
1927 };
1928 /* *INDENT-ON* */
1929
1930 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local);
1931
1932 static uword
1933 ip4_local_end_of_arc (vlib_main_t * vm,
1934                       vlib_node_runtime_t * node, vlib_frame_t * frame)
1935 {
1936   return ip4_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1937 }
1938
1939 /* *INDENT-OFF* */
1940 VLIB_REGISTER_NODE (ip4_local_end_of_arc_node,static) = {
1941   .function = ip4_local_end_of_arc,
1942   .name = "ip4-local-end-of-arc",
1943   .vector_size = sizeof (u32),
1944
1945   .format_trace = format_ip4_forward_next_trace,
1946   .sibling_of = "ip4-local",
1947 };
1948
1949 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_end_of_arc_node, ip4_local_end_of_arc)
1950
1951 VNET_FEATURE_INIT (ip4_local_end_of_arc, static) = {
1952   .arc_name = "ip4-local",
1953   .node_name = "ip4-local-end-of-arc",
1954   .runs_before = 0, /* not before any other features */
1955 };
1956 /* *INDENT-ON* */
1957
1958 void
1959 ip4_register_protocol (u32 protocol, u32 node_index)
1960 {
1961   vlib_main_t *vm = vlib_get_main ();
1962   ip4_main_t *im = &ip4_main;
1963   ip_lookup_main_t *lm = &im->lookup_main;
1964
1965   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1966   lm->local_next_by_ip_protocol[protocol] =
1967     vlib_node_add_next (vm, ip4_local_node.index, node_index);
1968 }
1969
1970 static clib_error_t *
1971 show_ip_local_command_fn (vlib_main_t * vm,
1972                           unformat_input_t * input, vlib_cli_command_t * cmd)
1973 {
1974   ip4_main_t *im = &ip4_main;
1975   ip_lookup_main_t *lm = &im->lookup_main;
1976   int i;
1977
1978   vlib_cli_output (vm, "Protocols handled by ip4_local");
1979   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
1980     {
1981       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
1982         vlib_cli_output (vm, "%d", i);
1983     }
1984   return 0;
1985 }
1986
1987
1988
1989 /*?
1990  * Display the set of protocols handled by the local IPv4 stack.
1991  *
1992  * @cliexpar
1993  * Example of how to display local protocol table:
1994  * @cliexstart{show ip local}
1995  * Protocols handled by ip4_local
1996  * 1
1997  * 17
1998  * 47
1999  * @cliexend
2000 ?*/
2001 /* *INDENT-OFF* */
2002 VLIB_CLI_COMMAND (show_ip_local, static) =
2003 {
2004   .path = "show ip local",
2005   .function = show_ip_local_command_fn,
2006   .short_help = "show ip local",
2007 };
2008 /* *INDENT-ON* */
2009
2010 always_inline uword
2011 ip4_arp_inline (vlib_main_t * vm,
2012                 vlib_node_runtime_t * node,
2013                 vlib_frame_t * frame, int is_glean)
2014 {
2015   vnet_main_t *vnm = vnet_get_main ();
2016   ip4_main_t *im = &ip4_main;
2017   ip_lookup_main_t *lm = &im->lookup_main;
2018   u32 *from, *to_next_drop;
2019   uword n_left_from, n_left_to_next_drop, next_index;
2020   static f64 time_last_seed_change = -1e100;
2021   static u32 hash_seeds[3];
2022   static uword hash_bitmap[256 / BITS (uword)];
2023   f64 time_now;
2024
2025   if (node->flags & VLIB_NODE_FLAG_TRACE)
2026     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2027
2028   time_now = vlib_time_now (vm);
2029   if (time_now - time_last_seed_change > 1e-3)
2030     {
2031       uword i;
2032       u32 *r = clib_random_buffer_get_data (&vm->random_buffer,
2033                                             sizeof (hash_seeds));
2034       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
2035         hash_seeds[i] = r[i];
2036
2037       /* Mark all hash keys as been no-seen before. */
2038       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
2039         hash_bitmap[i] = 0;
2040
2041       time_last_seed_change = time_now;
2042     }
2043
2044   from = vlib_frame_vector_args (frame);
2045   n_left_from = frame->n_vectors;
2046   next_index = node->cached_next_index;
2047   if (next_index == IP4_ARP_NEXT_DROP)
2048     next_index = IP4_ARP_N_NEXT;        /* point to first interface */
2049
2050   while (n_left_from > 0)
2051     {
2052       vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
2053                            to_next_drop, n_left_to_next_drop);
2054
2055       while (n_left_from > 0 && n_left_to_next_drop > 0)
2056         {
2057           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
2058           ip_adjacency_t *adj0;
2059           vlib_buffer_t *p0;
2060           ip4_header_t *ip0;
2061           uword bm0;
2062
2063           pi0 = from[0];
2064
2065           p0 = vlib_get_buffer (vm, pi0);
2066
2067           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2068           adj0 = adj_get (adj_index0);
2069           ip0 = vlib_buffer_get_current (p0);
2070
2071           a0 = hash_seeds[0];
2072           b0 = hash_seeds[1];
2073           c0 = hash_seeds[2];
2074
2075           sw_if_index0 = adj0->rewrite_header.sw_if_index;
2076           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
2077
2078           if (is_glean)
2079             {
2080               /*
2081                * this is the Glean case, so we are ARPing for the
2082                * packet's destination
2083                */
2084               a0 ^= ip0->dst_address.data_u32;
2085             }
2086           else
2087             {
2088               a0 ^= adj0->sub_type.nbr.next_hop.ip4.data_u32;
2089             }
2090           b0 ^= sw_if_index0;
2091
2092           hash_v3_finalize32 (a0, b0, c0);
2093
2094           c0 &= BITS (hash_bitmap) - 1;
2095           c0 = c0 / BITS (uword);
2096           m0 = (uword) 1 << (c0 % BITS (uword));
2097
2098           bm0 = hash_bitmap[c0];
2099           drop0 = (bm0 & m0) != 0;
2100
2101           /* Mark it as seen. */
2102           hash_bitmap[c0] = bm0 | m0;
2103
2104           from += 1;
2105           n_left_from -= 1;
2106           to_next_drop[0] = pi0;
2107           to_next_drop += 1;
2108           n_left_to_next_drop -= 1;
2109
2110           p0->error =
2111             node->errors[drop0 ? IP4_ARP_ERROR_DROP :
2112                          IP4_ARP_ERROR_REQUEST_SENT];
2113
2114           /*
2115            * the adj has been updated to a rewrite but the node the DPO that got
2116            * us here hasn't - yet. no big deal. we'll drop while we wait.
2117            */
2118           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
2119             continue;
2120
2121           if (drop0)
2122             continue;
2123
2124           /*
2125            * Can happen if the control-plane is programming tables
2126            * with traffic flowing; at least that's today's lame excuse.
2127            */
2128           if ((is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_GLEAN)
2129               || (!is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP))
2130             {
2131               p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
2132             }
2133           else
2134             /* Send ARP request. */
2135             {
2136               u32 bi0 = 0;
2137               vlib_buffer_t *b0;
2138               ethernet_arp_header_t *h0;
2139               vnet_hw_interface_t *hw_if0;
2140
2141               h0 =
2142                 vlib_packet_template_get_packet (vm,
2143                                                  &im->ip4_arp_request_packet_template,
2144                                                  &bi0);
2145
2146               /* Add rewrite/encap string for ARP packet. */
2147               vnet_rewrite_one_header (adj0[0], h0,
2148                                        sizeof (ethernet_header_t));
2149
2150               hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2151
2152               /* Src ethernet address in ARP header. */
2153               clib_memcpy (h0->ip4_over_ethernet[0].ethernet,
2154                            hw_if0->hw_address,
2155                            sizeof (h0->ip4_over_ethernet[0].ethernet));
2156
2157               if (is_glean)
2158                 {
2159                   /* The interface's source address is stashed in the Glean Adj */
2160                   h0->ip4_over_ethernet[0].ip4 =
2161                     adj0->sub_type.glean.receive_addr.ip4;
2162
2163                   /* Copy in destination address we are requesting. This is the
2164                    * glean case, so it's the packet's destination.*/
2165                   h0->ip4_over_ethernet[1].ip4.data_u32 =
2166                     ip0->dst_address.data_u32;
2167                 }
2168               else
2169                 {
2170                   /* Src IP address in ARP header. */
2171                   if (ip4_src_address_for_packet (lm, sw_if_index0,
2172                                                   &h0->
2173                                                   ip4_over_ethernet[0].ip4))
2174                     {
2175                       /* No source address available */
2176                       p0->error =
2177                         node->errors[IP4_ARP_ERROR_NO_SOURCE_ADDRESS];
2178                       vlib_buffer_free (vm, &bi0, 1);
2179                       continue;
2180                     }
2181
2182                   /* Copy in destination address we are requesting from the
2183                      incomplete adj */
2184                   h0->ip4_over_ethernet[1].ip4.data_u32 =
2185                     adj0->sub_type.nbr.next_hop.ip4.as_u32;
2186                 }
2187
2188               vlib_buffer_copy_trace_flag (vm, p0, bi0);
2189               b0 = vlib_get_buffer (vm, bi0);
2190               vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
2191
2192               vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2193
2194               vlib_set_next_frame_buffer (vm, node,
2195                                           adj0->rewrite_header.next_index,
2196                                           bi0);
2197             }
2198         }
2199
2200       vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
2201     }
2202
2203   return frame->n_vectors;
2204 }
2205
2206 static uword
2207 ip4_arp (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
2208 {
2209   return (ip4_arp_inline (vm, node, frame, 0));
2210 }
2211
2212 static uword
2213 ip4_glean (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
2214 {
2215   return (ip4_arp_inline (vm, node, frame, 1));
2216 }
2217
2218 static char *ip4_arp_error_strings[] = {
2219   [IP4_ARP_ERROR_DROP] = "address overflow drops",
2220   [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
2221   [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
2222   [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
2223   [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
2224   [IP4_ARP_ERROR_NO_SOURCE_ADDRESS] = "no source address for ARP request",
2225 };
2226
2227 VLIB_REGISTER_NODE (ip4_arp_node) =
2228 {
2229   .function = ip4_arp,.name = "ip4-arp",.vector_size =
2230     sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_errors =
2231     ARRAY_LEN (ip4_arp_error_strings),.error_strings =
2232     ip4_arp_error_strings,.n_next_nodes = IP4_ARP_N_NEXT,.next_nodes =
2233   {
2234   [IP4_ARP_NEXT_DROP] = "error-drop",}
2235 ,};
2236
2237 VLIB_REGISTER_NODE (ip4_glean_node) =
2238 {
2239   .function = ip4_glean,.name = "ip4-glean",.vector_size =
2240     sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_errors =
2241     ARRAY_LEN (ip4_arp_error_strings),.error_strings =
2242     ip4_arp_error_strings,.n_next_nodes = IP4_ARP_N_NEXT,.next_nodes =
2243   {
2244   [IP4_ARP_NEXT_DROP] = "error-drop",}
2245 ,};
2246
2247 #define foreach_notrace_ip4_arp_error           \
2248 _(DROP)                                         \
2249 _(REQUEST_SENT)                                 \
2250 _(REPLICATE_DROP)                               \
2251 _(REPLICATE_FAIL)
2252
2253 clib_error_t *
2254 arp_notrace_init (vlib_main_t * vm)
2255 {
2256   vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, ip4_arp_node.index);
2257
2258   /* don't trace ARP request packets */
2259 #define _(a)                                    \
2260     vnet_pcap_drop_trace_filter_add_del         \
2261         (rt->errors[IP4_ARP_ERROR_##a],         \
2262          1 /* is_add */);
2263   foreach_notrace_ip4_arp_error;
2264 #undef _
2265   return 0;
2266 }
2267
2268 VLIB_INIT_FUNCTION (arp_notrace_init);
2269
2270
2271 /* Send an ARP request to see if given destination is reachable on given interface. */
2272 clib_error_t *
2273 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
2274 {
2275   vnet_main_t *vnm = vnet_get_main ();
2276   ip4_main_t *im = &ip4_main;
2277   ethernet_arp_header_t *h;
2278   ip4_address_t *src;
2279   ip_interface_address_t *ia;
2280   ip_adjacency_t *adj;
2281   vnet_hw_interface_t *hi;
2282   vnet_sw_interface_t *si;
2283   vlib_buffer_t *b;
2284   adj_index_t ai;
2285   u32 bi = 0;
2286
2287   si = vnet_get_sw_interface (vnm, sw_if_index);
2288
2289   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2290     {
2291       return clib_error_return (0, "%U: interface %U down",
2292                                 format_ip4_address, dst,
2293                                 format_vnet_sw_if_index_name, vnm,
2294                                 sw_if_index);
2295     }
2296
2297   src =
2298     ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2299   if (!src)
2300     {
2301       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2302       return clib_error_return
2303         (0,
2304          "no matching interface address for destination %U (interface %U)",
2305          format_ip4_address, dst, format_vnet_sw_if_index_name, vnm,
2306          sw_if_index);
2307     }
2308
2309   ip46_address_t nh = {
2310     .ip4 = *dst,
2311   };
2312
2313   ai = adj_nbr_add_or_lock (FIB_PROTOCOL_IP4,
2314                             VNET_LINK_IP4, &nh, sw_if_index);
2315   adj = adj_get (ai);
2316
2317   h = vlib_packet_template_get_packet (vm,
2318                                        &im->ip4_arp_request_packet_template,
2319                                        &bi);
2320
2321   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2322
2323   clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address,
2324                sizeof (h->ip4_over_ethernet[0].ethernet));
2325
2326   h->ip4_over_ethernet[0].ip4 = src[0];
2327   h->ip4_over_ethernet[1].ip4 = dst[0];
2328
2329   b = vlib_get_buffer (vm, bi);
2330   vnet_buffer (b)->sw_if_index[VLIB_RX] =
2331     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2332
2333   /* Add encapsulation string for software interface (e.g. ethernet header). */
2334   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2335   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2336
2337   {
2338     vlib_frame_t *f = vlib_get_frame_to_node (vm, hi->output_node_index);
2339     u32 *to_next = vlib_frame_vector_args (f);
2340     to_next[0] = bi;
2341     f->n_vectors = 1;
2342     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2343   }
2344
2345   adj_unlock (ai);
2346   return /* no error */ 0;
2347 }
2348
2349 typedef enum
2350 {
2351   IP4_REWRITE_NEXT_DROP,
2352   IP4_REWRITE_NEXT_ICMP_ERROR,
2353 } ip4_rewrite_next_t;
2354
2355 always_inline uword
2356 ip4_rewrite_inline (vlib_main_t * vm,
2357                     vlib_node_runtime_t * node,
2358                     vlib_frame_t * frame,
2359                     int do_counters, int is_midchain, int is_mcast)
2360 {
2361   ip_lookup_main_t *lm = &ip4_main.lookup_main;
2362   u32 *from = vlib_frame_vector_args (frame);
2363   u32 n_left_from, n_left_to_next, *to_next, next_index;
2364   vlib_node_runtime_t *error_node =
2365     vlib_node_get_runtime (vm, ip4_input_node.index);
2366
2367   n_left_from = frame->n_vectors;
2368   next_index = node->cached_next_index;
2369   u32 thread_index = vlib_get_thread_index ();
2370
2371   while (n_left_from > 0)
2372     {
2373       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2374
2375       while (n_left_from >= 4 && n_left_to_next >= 2)
2376         {
2377           ip_adjacency_t *adj0, *adj1;
2378           vlib_buffer_t *p0, *p1;
2379           ip4_header_t *ip0, *ip1;
2380           u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
2381           u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
2382           u32 tx_sw_if_index0, tx_sw_if_index1;
2383
2384           /* Prefetch next iteration. */
2385           {
2386             vlib_buffer_t *p2, *p3;
2387
2388             p2 = vlib_get_buffer (vm, from[2]);
2389             p3 = vlib_get_buffer (vm, from[3]);
2390
2391             vlib_prefetch_buffer_header (p2, STORE);
2392             vlib_prefetch_buffer_header (p3, STORE);
2393
2394             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2395             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2396           }
2397
2398           pi0 = to_next[0] = from[0];
2399           pi1 = to_next[1] = from[1];
2400
2401           from += 2;
2402           n_left_from -= 2;
2403           to_next += 2;
2404           n_left_to_next -= 2;
2405
2406           p0 = vlib_get_buffer (vm, pi0);
2407           p1 = vlib_get_buffer (vm, pi1);
2408
2409           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2410           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
2411
2412           /*
2413            * pre-fetch the per-adjacency counters
2414            */
2415           if (do_counters)
2416             {
2417               vlib_prefetch_combined_counter (&adjacency_counters,
2418                                               thread_index, adj_index0);
2419               vlib_prefetch_combined_counter (&adjacency_counters,
2420                                               thread_index, adj_index1);
2421             }
2422
2423           ip0 = vlib_buffer_get_current (p0);
2424           ip1 = vlib_buffer_get_current (p1);
2425
2426           error0 = error1 = IP4_ERROR_NONE;
2427           next0 = next1 = IP4_REWRITE_NEXT_DROP;
2428
2429           /* Decrement TTL & update checksum.
2430              Works either endian, so no need for byte swap. */
2431           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_LOCALLY_ORIGINATED)))
2432             {
2433               i32 ttl0 = ip0->ttl;
2434
2435               /* Input node should have reject packets with ttl 0. */
2436               ASSERT (ip0->ttl > 0);
2437
2438               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2439               checksum0 += checksum0 >= 0xffff;
2440
2441               ip0->checksum = checksum0;
2442               ttl0 -= 1;
2443               ip0->ttl = ttl0;
2444
2445               /*
2446                * If the ttl drops below 1 when forwarding, generate
2447                * an ICMP response.
2448                */
2449               if (PREDICT_FALSE (ttl0 <= 0))
2450                 {
2451                   error0 = IP4_ERROR_TIME_EXPIRED;
2452                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2453                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2454                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2455                                                0);
2456                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2457                 }
2458
2459               /* Verify checksum. */
2460               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2461             }
2462           else
2463             {
2464               p0->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
2465             }
2466           if (PREDICT_TRUE (!(p1->flags & VNET_BUFFER_LOCALLY_ORIGINATED)))
2467             {
2468               i32 ttl1 = ip1->ttl;
2469
2470               /* Input node should have reject packets with ttl 0. */
2471               ASSERT (ip1->ttl > 0);
2472
2473               checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2474               checksum1 += checksum1 >= 0xffff;
2475
2476               ip1->checksum = checksum1;
2477               ttl1 -= 1;
2478               ip1->ttl = ttl1;
2479
2480               /*
2481                * If the ttl drops below 1 when forwarding, generate
2482                * an ICMP response.
2483                */
2484               if (PREDICT_FALSE (ttl1 <= 0))
2485                 {
2486                   error1 = IP4_ERROR_TIME_EXPIRED;
2487                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2488                   icmp4_error_set_vnet_buffer (p1, ICMP4_time_exceeded,
2489                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2490                                                0);
2491                   next1 = IP4_REWRITE_NEXT_ICMP_ERROR;
2492                 }
2493
2494               /* Verify checksum. */
2495               ASSERT (ip1->checksum == ip4_header_checksum (ip1));
2496             }
2497           else
2498             {
2499               p1->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
2500             }
2501
2502           /* Rewrite packet header and updates lengths. */
2503           adj0 = adj_get (adj_index0);
2504           adj1 = adj_get (adj_index1);
2505
2506           /* Worth pipelining. No guarantee that adj0,1 are hot... */
2507           rw_len0 = adj0[0].rewrite_header.data_bytes;
2508           rw_len1 = adj1[0].rewrite_header.data_bytes;
2509           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2510           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
2511
2512           /* Check MTU of outgoing interface. */
2513           error0 =
2514             (vlib_buffer_length_in_chain (vm, p0) >
2515              adj0[0].
2516              rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
2517              error0);
2518           error1 =
2519             (vlib_buffer_length_in_chain (vm, p1) >
2520              adj1[0].
2521              rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
2522              error1);
2523
2524           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2525            * to see the IP headerr */
2526           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2527             {
2528               next0 = adj0[0].rewrite_header.next_index;
2529               p0->current_data -= rw_len0;
2530               p0->current_length += rw_len0;
2531               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2532               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2533
2534               if (PREDICT_FALSE
2535                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2536                 vnet_feature_arc_start (lm->output_feature_arc_index,
2537                                         tx_sw_if_index0, &next0, p0);
2538             }
2539           if (PREDICT_TRUE (error1 == IP4_ERROR_NONE))
2540             {
2541               next1 = adj1[0].rewrite_header.next_index;
2542               p1->current_data -= rw_len1;
2543               p1->current_length += rw_len1;
2544
2545               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
2546               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
2547
2548               if (PREDICT_FALSE
2549                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2550                 vnet_feature_arc_start (lm->output_feature_arc_index,
2551                                         tx_sw_if_index1, &next1, p1);
2552             }
2553
2554           /* Guess we are only writing on simple Ethernet header. */
2555           vnet_rewrite_two_headers (adj0[0], adj1[0],
2556                                     ip0, ip1, sizeof (ethernet_header_t));
2557
2558           /*
2559            * Bump the per-adjacency counters
2560            */
2561           if (do_counters)
2562             {
2563               vlib_increment_combined_counter
2564                 (&adjacency_counters,
2565                  thread_index,
2566                  adj_index0, 1,
2567                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2568
2569               vlib_increment_combined_counter
2570                 (&adjacency_counters,
2571                  thread_index,
2572                  adj_index1, 1,
2573                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
2574             }
2575
2576           if (is_midchain)
2577             {
2578               adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
2579               adj1->sub_type.midchain.fixup_func (vm, adj1, p1);
2580             }
2581           if (is_mcast)
2582             {
2583               /*
2584                * copy bytes from the IP address into the MAC rewrite
2585                */
2586               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2587               vnet_fixup_one_header (adj1[0], &ip1->dst_address, ip1);
2588             }
2589
2590           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2591                                            to_next, n_left_to_next,
2592                                            pi0, pi1, next0, next1);
2593         }
2594
2595       while (n_left_from > 0 && n_left_to_next > 0)
2596         {
2597           ip_adjacency_t *adj0;
2598           vlib_buffer_t *p0;
2599           ip4_header_t *ip0;
2600           u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2601           u32 tx_sw_if_index0;
2602
2603           pi0 = to_next[0] = from[0];
2604
2605           p0 = vlib_get_buffer (vm, pi0);
2606
2607           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2608
2609           adj0 = adj_get (adj_index0);
2610
2611           ip0 = vlib_buffer_get_current (p0);
2612
2613           error0 = IP4_ERROR_NONE;
2614           next0 = IP4_REWRITE_NEXT_DROP;        /* drop on error */
2615
2616           /* Decrement TTL & update checksum. */
2617           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_LOCALLY_ORIGINATED)))
2618             {
2619               i32 ttl0 = ip0->ttl;
2620
2621               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2622
2623               checksum0 += checksum0 >= 0xffff;
2624
2625               ip0->checksum = checksum0;
2626
2627               ASSERT (ip0->ttl > 0);
2628
2629               ttl0 -= 1;
2630
2631               ip0->ttl = ttl0;
2632
2633               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2634
2635               if (PREDICT_FALSE (ttl0 <= 0))
2636                 {
2637                   /*
2638                    * If the ttl drops below 1 when forwarding, generate
2639                    * an ICMP response.
2640                    */
2641                   error0 = IP4_ERROR_TIME_EXPIRED;
2642                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2643                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2644                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2645                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2646                                                0);
2647                 }
2648             }
2649           else
2650             {
2651               p0->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
2652             }
2653
2654           if (do_counters)
2655             vlib_prefetch_combined_counter (&adjacency_counters,
2656                                             thread_index, adj_index0);
2657
2658           /* Guess we are only writing on simple Ethernet header. */
2659           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2660           if (is_mcast)
2661             {
2662               /*
2663                * copy bytes from the IP address into the MAC rewrite
2664                */
2665               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2666             }
2667
2668           /* Update packet buffer attributes/set output interface. */
2669           rw_len0 = adj0[0].rewrite_header.data_bytes;
2670           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2671
2672           if (do_counters)
2673             vlib_increment_combined_counter
2674               (&adjacency_counters,
2675                thread_index, adj_index0, 1,
2676                vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2677
2678           /* Check MTU of outgoing interface. */
2679           error0 = (vlib_buffer_length_in_chain (vm, p0)
2680                     > adj0[0].rewrite_header.max_l3_packet_bytes
2681                     ? IP4_ERROR_MTU_EXCEEDED : error0);
2682
2683           p0->error = error_node->errors[error0];
2684
2685           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2686            * to see the IP headerr */
2687           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2688             {
2689               p0->current_data -= rw_len0;
2690               p0->current_length += rw_len0;
2691               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2692
2693               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2694               next0 = adj0[0].rewrite_header.next_index;
2695
2696               if (is_midchain)
2697                 {
2698                   adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
2699                 }
2700
2701               if (PREDICT_FALSE
2702                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2703                 vnet_feature_arc_start (lm->output_feature_arc_index,
2704                                         tx_sw_if_index0, &next0, p0);
2705
2706             }
2707
2708           from += 1;
2709           n_left_from -= 1;
2710           to_next += 1;
2711           n_left_to_next -= 1;
2712
2713           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2714                                            to_next, n_left_to_next,
2715                                            pi0, next0);
2716         }
2717
2718       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2719     }
2720
2721   /* Need to do trace after rewrites to pick up new packet data. */
2722   if (node->flags & VLIB_NODE_FLAG_TRACE)
2723     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2724
2725   return frame->n_vectors;
2726 }
2727
2728
2729 /** @brief IPv4 rewrite node.
2730     @node ip4-rewrite
2731
2732     This is the IPv4 transit-rewrite node: decrement TTL, fix the ipv4
2733     header checksum, fetch the ip adjacency, check the outbound mtu,
2734     apply the adjacency rewrite, and send pkts to the adjacency
2735     rewrite header's rewrite_next_index.
2736
2737     @param vm vlib_main_t corresponding to the current thread
2738     @param node vlib_node_runtime_t
2739     @param frame vlib_frame_t whose contents should be dispatched
2740
2741     @par Graph mechanics: buffer metadata, next index usage
2742
2743     @em Uses:
2744     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
2745         - the rewrite adjacency index
2746     - <code>adj->lookup_next_index</code>
2747         - Must be IP_LOOKUP_NEXT_REWRITE or IP_LOOKUP_NEXT_ARP, otherwise
2748           the packet will be dropped.
2749     - <code>adj->rewrite_header</code>
2750         - Rewrite string length, rewrite string, next_index
2751
2752     @em Sets:
2753     - <code>b->current_data, b->current_length</code>
2754         - Updated net of applying the rewrite string
2755
2756     <em>Next Indices:</em>
2757     - <code> adj->rewrite_header.next_index </code>
2758       or @c error-drop
2759 */
2760 static uword
2761 ip4_rewrite (vlib_main_t * vm,
2762              vlib_node_runtime_t * node, vlib_frame_t * frame)
2763 {
2764   if (adj_are_counters_enabled ())
2765     return ip4_rewrite_inline (vm, node, frame, 1, 0, 0);
2766   else
2767     return ip4_rewrite_inline (vm, node, frame, 0, 0, 0);
2768 }
2769
2770 static uword
2771 ip4_midchain (vlib_main_t * vm,
2772               vlib_node_runtime_t * node, vlib_frame_t * frame)
2773 {
2774   if (adj_are_counters_enabled ())
2775     return ip4_rewrite_inline (vm, node, frame, 1, 1, 0);
2776   else
2777     return ip4_rewrite_inline (vm, node, frame, 0, 1, 0);
2778 }
2779
2780 static uword
2781 ip4_rewrite_mcast (vlib_main_t * vm,
2782                    vlib_node_runtime_t * node, vlib_frame_t * frame)
2783 {
2784   if (adj_are_counters_enabled ())
2785     return ip4_rewrite_inline (vm, node, frame, 1, 0, 1);
2786   else
2787     return ip4_rewrite_inline (vm, node, frame, 0, 0, 1);
2788 }
2789
2790 static uword
2791 ip4_mcast_midchain (vlib_main_t * vm,
2792                     vlib_node_runtime_t * node, vlib_frame_t * frame)
2793 {
2794   if (adj_are_counters_enabled ())
2795     return ip4_rewrite_inline (vm, node, frame, 1, 1, 1);
2796   else
2797     return ip4_rewrite_inline (vm, node, frame, 0, 1, 1);
2798 }
2799
2800 /* *INDENT-OFF* */
2801 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2802   .function = ip4_rewrite,
2803   .name = "ip4-rewrite",
2804   .vector_size = sizeof (u32),
2805
2806   .format_trace = format_ip4_rewrite_trace,
2807
2808   .n_next_nodes = 2,
2809   .next_nodes = {
2810     [IP4_REWRITE_NEXT_DROP] = "error-drop",
2811     [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2812   },
2813 };
2814 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite)
2815
2816 VLIB_REGISTER_NODE (ip4_rewrite_mcast_node) = {
2817   .function = ip4_rewrite_mcast,
2818   .name = "ip4-rewrite-mcast",
2819   .vector_size = sizeof (u32),
2820
2821   .format_trace = format_ip4_rewrite_trace,
2822   .sibling_of = "ip4-rewrite",
2823 };
2824 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_mcast_node, ip4_rewrite_mcast)
2825
2826 VLIB_REGISTER_NODE (ip4_mcast_midchain_node, static) = {
2827   .function = ip4_mcast_midchain,
2828   .name = "ip4-mcast-midchain",
2829   .vector_size = sizeof (u32),
2830
2831   .format_trace = format_ip4_rewrite_trace,
2832   .sibling_of = "ip4-rewrite",
2833 };
2834 VLIB_NODE_FUNCTION_MULTIARCH (ip4_mcast_midchain_node, ip4_mcast_midchain)
2835
2836 VLIB_REGISTER_NODE (ip4_midchain_node) = {
2837   .function = ip4_midchain,
2838   .name = "ip4-midchain",
2839   .vector_size = sizeof (u32),
2840   .format_trace = format_ip4_forward_next_trace,
2841   .sibling_of =  "ip4-rewrite",
2842 };
2843 VLIB_NODE_FUNCTION_MULTIARCH (ip4_midchain_node, ip4_midchain);
2844 /* *INDENT-ON */
2845
2846 static clib_error_t *
2847 add_del_interface_table (vlib_main_t * vm,
2848                          unformat_input_t * input, vlib_cli_command_t * cmd)
2849 {
2850   vnet_main_t *vnm = vnet_get_main ();
2851   ip_interface_address_t *ia;
2852   clib_error_t *error = 0;
2853   u32 sw_if_index, table_id;
2854
2855   sw_if_index = ~0;
2856
2857   if (!unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2858     {
2859       error = clib_error_return (0, "unknown interface `%U'",
2860                                  format_unformat_error, input);
2861       goto done;
2862     }
2863
2864   if (unformat (input, "%d", &table_id))
2865     ;
2866   else
2867     {
2868       error = clib_error_return (0, "expected table id `%U'",
2869                                  format_unformat_error, input);
2870       goto done;
2871     }
2872
2873   /*
2874    * If the interface already has in IP address, then a change int
2875    * VRF is not allowed. The IP address applied must first be removed.
2876    * We do not do that automatically here, since VPP has no knowledge
2877    * of whether thoses subnets are valid in the destination VRF.
2878    */
2879   /* *INDENT-OFF* */
2880   foreach_ip_interface_address (&ip4_main.lookup_main,
2881                                 ia, sw_if_index,
2882                                 1 /* honor unnumbered */,
2883   ({
2884       ip4_address_t * a;
2885
2886       a = ip_interface_address_get_address (&ip4_main.lookup_main, ia);
2887       error = clib_error_return (0, "interface %U has address %U",
2888                                  format_vnet_sw_if_index_name, vnm,
2889                                  sw_if_index,
2890                                  format_ip4_address, a);
2891       goto done;
2892    }));
2893    /* *INDENT-ON* */
2894
2895 {
2896   ip4_main_t *im = &ip4_main;
2897   u32 fib_index;
2898
2899   fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, table_id);
2900
2901   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
2902   im->fib_index_by_sw_if_index[sw_if_index] = fib_index;
2903
2904   fib_index = mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, table_id);
2905   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
2906   im->mfib_index_by_sw_if_index[sw_if_index] = fib_index;
2907 }
2908
2909 done:
2910 return error;
2911 }
2912
2913 /*?
2914  * Place the indicated interface into the supplied IPv4 FIB table (also known
2915  * as a VRF). If the FIB table does not exist, this command creates it. To
2916  * display the current IPv4 FIB table, use the command '<em>show ip fib</em>'.
2917  * FIB table will only be displayed if a route has been added to the table, or
2918  * an IP Address is assigned to an interface in the table (which adds a route
2919  * automatically).
2920  *
2921  * @note IP addresses added after setting the interface IP table are added to
2922  * the indicated FIB table. If an IP address is added prior to changing the
2923  * table then this is an error. The control plane must remove these addresses
2924  * first and then change the table. VPP will not automatically move the
2925  * addresses from the old to the new table as it does not know the validity
2926  * of such a change.
2927  *
2928  * @cliexpar
2929  * Example of how to add an interface to an IPv4 FIB table (where 2 is the table-id):
2930  * @cliexcmd{set interface ip table GigabitEthernet2/0/0 2}
2931  ?*/
2932 /* *INDENT-OFF* */
2933 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) =
2934 {
2935   .path = "set interface ip table",
2936   .function = add_del_interface_table,
2937   .short_help = "set interface ip table <interface> <table-id>",
2938 };
2939 /* *INDENT-ON* */
2940
2941 int
2942 ip4_lookup_validate (ip4_address_t * a, u32 fib_index0)
2943 {
2944   ip4_fib_mtrie_t *mtrie0;
2945   ip4_fib_mtrie_leaf_t leaf0;
2946   u32 lbi0;
2947
2948   mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
2949
2950   leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, a);
2951   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
2952   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
2953
2954   lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2955
2956   return lbi0 == ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0), a);
2957 }
2958
2959 static clib_error_t *
2960 test_lookup_command_fn (vlib_main_t * vm,
2961                         unformat_input_t * input, vlib_cli_command_t * cmd)
2962 {
2963   ip4_fib_t *fib;
2964   u32 table_id = 0;
2965   f64 count = 1;
2966   u32 n;
2967   int i;
2968   ip4_address_t ip4_base_address;
2969   u64 errors = 0;
2970
2971   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2972     {
2973       if (unformat (input, "table %d", &table_id))
2974         {
2975           /* Make sure the entry exists. */
2976           fib = ip4_fib_get (table_id);
2977           if ((fib) && (fib->index != table_id))
2978             return clib_error_return (0, "<fib-index> %d does not exist",
2979                                       table_id);
2980         }
2981       else if (unformat (input, "count %f", &count))
2982         ;
2983
2984       else if (unformat (input, "%U",
2985                          unformat_ip4_address, &ip4_base_address))
2986         ;
2987       else
2988         return clib_error_return (0, "unknown input `%U'",
2989                                   format_unformat_error, input);
2990     }
2991
2992   n = count;
2993
2994   for (i = 0; i < n; i++)
2995     {
2996       if (!ip4_lookup_validate (&ip4_base_address, table_id))
2997         errors++;
2998
2999       ip4_base_address.as_u32 =
3000         clib_host_to_net_u32 (1 +
3001                               clib_net_to_host_u32 (ip4_base_address.as_u32));
3002     }
3003
3004   if (errors)
3005     vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
3006   else
3007     vlib_cli_output (vm, "No errors in %d lookups\n", n);
3008
3009   return 0;
3010 }
3011
3012 /*?
3013  * Perform a lookup of an IPv4 Address (or range of addresses) in the
3014  * given FIB table to determine if there is a conflict with the
3015  * adjacency table. The fib-id can be determined by using the
3016  * '<em>show ip fib</em>' command. If fib-id is not entered, default value
3017  * of 0 is used.
3018  *
3019  * @todo This command uses fib-id, other commands use table-id (not
3020  * just a name, they are different indexes). Would like to change this
3021  * to table-id for consistency.
3022  *
3023  * @cliexpar
3024  * Example of how to run the test lookup command:
3025  * @cliexstart{test lookup 172.16.1.1 table 1 count 2}
3026  * No errors in 2 lookups
3027  * @cliexend
3028 ?*/
3029 /* *INDENT-OFF* */
3030 VLIB_CLI_COMMAND (lookup_test_command, static) =
3031 {
3032   .path = "test lookup",
3033   .short_help = "test lookup <ipv4-addr> [table <fib-id>] [count <nn>]",
3034   .function = test_lookup_command_fn,
3035 };
3036 /* *INDENT-ON* */
3037
3038 int
3039 vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
3040 {
3041   u32 fib_index;
3042
3043   fib_index = fib_table_find (FIB_PROTOCOL_IP4, table_id);
3044
3045   if (~0 == fib_index)
3046     return VNET_API_ERROR_NO_SUCH_FIB;
3047
3048   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP4,
3049                                   flow_hash_config);
3050
3051   return 0;
3052 }
3053
3054 static clib_error_t *
3055 set_ip_flow_hash_command_fn (vlib_main_t * vm,
3056                              unformat_input_t * input,
3057                              vlib_cli_command_t * cmd)
3058 {
3059   int matched = 0;
3060   u32 table_id = 0;
3061   u32 flow_hash_config = 0;
3062   int rv;
3063
3064   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3065     {
3066       if (unformat (input, "table %d", &table_id))
3067         matched = 1;
3068 #define _(a,v) \
3069     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
3070       foreach_flow_hash_bit
3071 #undef _
3072         else
3073         break;
3074     }
3075
3076   if (matched == 0)
3077     return clib_error_return (0, "unknown input `%U'",
3078                               format_unformat_error, input);
3079
3080   rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
3081   switch (rv)
3082     {
3083     case 0:
3084       break;
3085
3086     case VNET_API_ERROR_NO_SUCH_FIB:
3087       return clib_error_return (0, "no such FIB table %d", table_id);
3088
3089     default:
3090       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3091       break;
3092     }
3093
3094   return 0;
3095 }
3096
3097 /*?
3098  * Configure the set of IPv4 fields used by the flow hash.
3099  *
3100  * @cliexpar
3101  * Example of how to set the flow hash on a given table:
3102  * @cliexcmd{set ip flow-hash table 7 dst sport dport proto}
3103  * Example of display the configured flow hash:
3104  * @cliexstart{show ip fib}
3105  * ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
3106  * 0.0.0.0/0
3107  *   unicast-ip4-chain
3108  *   [@0]: dpo-load-balance: [index:0 buckets:1 uRPF:0 to:[0:0]]
3109  *     [0] [@0]: dpo-drop ip6
3110  * 0.0.0.0/32
3111  *   unicast-ip4-chain
3112  *   [@0]: dpo-load-balance: [index:1 buckets:1 uRPF:1 to:[0:0]]
3113  *     [0] [@0]: dpo-drop ip6
3114  * 224.0.0.0/8
3115  *   unicast-ip4-chain
3116  *   [@0]: dpo-load-balance: [index:3 buckets:1 uRPF:3 to:[0:0]]
3117  *     [0] [@0]: dpo-drop ip6
3118  * 6.0.1.2/32
3119  *   unicast-ip4-chain
3120  *   [@0]: dpo-load-balance: [index:30 buckets:1 uRPF:29 to:[0:0]]
3121  *     [0] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
3122  * 7.0.0.1/32
3123  *   unicast-ip4-chain
3124  *   [@0]: dpo-load-balance: [index:31 buckets:4 uRPF:30 to:[0:0]]
3125  *     [0] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3126  *     [1] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3127  *     [2] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3128  *     [3] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
3129  * 240.0.0.0/8
3130  *   unicast-ip4-chain
3131  *   [@0]: dpo-load-balance: [index:2 buckets:1 uRPF:2 to:[0:0]]
3132  *     [0] [@0]: dpo-drop ip6
3133  * 255.255.255.255/32
3134  *   unicast-ip4-chain
3135  *   [@0]: dpo-load-balance: [index:4 buckets:1 uRPF:4 to:[0:0]]
3136  *     [0] [@0]: dpo-drop ip6
3137  * ipv4-VRF:7, fib_index 1, flow hash: dst sport dport proto
3138  * 0.0.0.0/0
3139  *   unicast-ip4-chain
3140  *   [@0]: dpo-load-balance: [index:12 buckets:1 uRPF:11 to:[0:0]]
3141  *     [0] [@0]: dpo-drop ip6
3142  * 0.0.0.0/32
3143  *   unicast-ip4-chain
3144  *   [@0]: dpo-load-balance: [index:13 buckets:1 uRPF:12 to:[0:0]]
3145  *     [0] [@0]: dpo-drop ip6
3146  * 172.16.1.0/24
3147  *   unicast-ip4-chain
3148  *   [@0]: dpo-load-balance: [index:17 buckets:1 uRPF:16 to:[0:0]]
3149  *     [0] [@4]: ipv4-glean: af_packet0
3150  * 172.16.1.1/32
3151  *   unicast-ip4-chain
3152  *   [@0]: dpo-load-balance: [index:18 buckets:1 uRPF:17 to:[1:84]]
3153  *     [0] [@2]: dpo-receive: 172.16.1.1 on af_packet0
3154  * 172.16.1.2/32
3155  *   unicast-ip4-chain
3156  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
3157  *     [0] [@5]: ipv4 via 172.16.1.2 af_packet0: IP4: 02:fe:9e:70:7a:2b -> 26:a5:f6:9c:3a:36
3158  * 172.16.2.0/24
3159  *   unicast-ip4-chain
3160  *   [@0]: dpo-load-balance: [index:19 buckets:1 uRPF:18 to:[0:0]]
3161  *     [0] [@4]: ipv4-glean: af_packet1
3162  * 172.16.2.1/32
3163  *   unicast-ip4-chain
3164  *   [@0]: dpo-load-balance: [index:20 buckets:1 uRPF:19 to:[0:0]]
3165  *     [0] [@2]: dpo-receive: 172.16.2.1 on af_packet1
3166  * 224.0.0.0/8
3167  *   unicast-ip4-chain
3168  *   [@0]: dpo-load-balance: [index:15 buckets:1 uRPF:14 to:[0:0]]
3169  *     [0] [@0]: dpo-drop ip6
3170  * 240.0.0.0/8
3171  *   unicast-ip4-chain
3172  *   [@0]: dpo-load-balance: [index:14 buckets:1 uRPF:13 to:[0:0]]
3173  *     [0] [@0]: dpo-drop ip6
3174  * 255.255.255.255/32
3175  *   unicast-ip4-chain
3176  *   [@0]: dpo-load-balance: [index:16 buckets:1 uRPF:15 to:[0:0]]
3177  *     [0] [@0]: dpo-drop ip6
3178  * @cliexend
3179 ?*/
3180 /* *INDENT-OFF* */
3181 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) =
3182 {
3183   .path = "set ip flow-hash",
3184   .short_help =
3185   "set ip flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
3186   .function = set_ip_flow_hash_command_fn,
3187 };
3188 /* *INDENT-ON* */
3189
3190 int
3191 vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3192                              u32 table_index)
3193 {
3194   vnet_main_t *vnm = vnet_get_main ();
3195   vnet_interface_main_t *im = &vnm->interface_main;
3196   ip4_main_t *ipm = &ip4_main;
3197   ip_lookup_main_t *lm = &ipm->lookup_main;
3198   vnet_classify_main_t *cm = &vnet_classify_main;
3199   ip4_address_t *if_addr;
3200
3201   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3202     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3203
3204   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3205     return VNET_API_ERROR_NO_SUCH_ENTRY;
3206
3207   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3208   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
3209
3210   if_addr = ip4_interface_first_address (ipm, sw_if_index, NULL);
3211
3212   if (NULL != if_addr)
3213     {
3214       fib_prefix_t pfx = {
3215         .fp_len = 32,
3216         .fp_proto = FIB_PROTOCOL_IP4,
3217         .fp_addr.ip4 = *if_addr,
3218       };
3219       u32 fib_index;
3220
3221       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3222                                                        sw_if_index);
3223
3224
3225       if (table_index != (u32) ~ 0)
3226         {
3227           dpo_id_t dpo = DPO_INVALID;
3228
3229           dpo_set (&dpo,
3230                    DPO_CLASSIFY,
3231                    DPO_PROTO_IP4,
3232                    classify_dpo_create (DPO_PROTO_IP4, table_index));
3233
3234           fib_table_entry_special_dpo_add (fib_index,
3235                                            &pfx,
3236                                            FIB_SOURCE_CLASSIFY,
3237                                            FIB_ENTRY_FLAG_NONE, &dpo);
3238           dpo_reset (&dpo);
3239         }
3240       else
3241         {
3242           fib_table_entry_special_remove (fib_index,
3243                                           &pfx, FIB_SOURCE_CLASSIFY);
3244         }
3245     }
3246
3247   return 0;
3248 }
3249
3250 static clib_error_t *
3251 set_ip_classify_command_fn (vlib_main_t * vm,
3252                             unformat_input_t * input,
3253                             vlib_cli_command_t * cmd)
3254 {
3255   u32 table_index = ~0;
3256   int table_index_set = 0;
3257   u32 sw_if_index = ~0;
3258   int rv;
3259
3260   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3261     {
3262       if (unformat (input, "table-index %d", &table_index))
3263         table_index_set = 1;
3264       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3265                          vnet_get_main (), &sw_if_index))
3266         ;
3267       else
3268         break;
3269     }
3270
3271   if (table_index_set == 0)
3272     return clib_error_return (0, "classify table-index must be specified");
3273
3274   if (sw_if_index == ~0)
3275     return clib_error_return (0, "interface / subif must be specified");
3276
3277   rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
3278
3279   switch (rv)
3280     {
3281     case 0:
3282       break;
3283
3284     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3285       return clib_error_return (0, "No such interface");
3286
3287     case VNET_API_ERROR_NO_SUCH_ENTRY:
3288       return clib_error_return (0, "No such classifier table");
3289     }
3290   return 0;
3291 }
3292
3293 /*?
3294  * Assign a classification table to an interface. The classification
3295  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3296  * commands. Once the table is create, use this command to filter packets
3297  * on an interface.
3298  *
3299  * @cliexpar
3300  * Example of how to assign a classification table to an interface:
3301  * @cliexcmd{set ip classify intfc GigabitEthernet2/0/0 table-index 1}
3302 ?*/
3303 /* *INDENT-OFF* */
3304 VLIB_CLI_COMMAND (set_ip_classify_command, static) =
3305 {
3306     .path = "set ip classify",
3307     .short_help =
3308     "set ip classify intfc <interface> table-index <classify-idx>",
3309     .function = set_ip_classify_command_fn,
3310 };
3311 /* *INDENT-ON* */
3312
3313 /*
3314  * fd.io coding-style-patch-verification: ON
3315  *
3316  * Local Variables:
3317  * eval: (c-set-style "gnu")
3318  * End:
3319  */