src/vnet/ip/ip4_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip4_forward.c: IP v4 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  43 #include <vnet/ethernet/arp_packet.h>   /* for ethernet_arp_header_t */
  44 #include <vnet/ppp/ppp.h>
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vnet/api_errno.h>     /* for API error numbers */
  47 #include <vnet/fib/fib_table.h> /* for FIB table and entry creation */
  48 #include <vnet/fib/fib_entry.h> /* for FIB table and entry creation */
  49 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  50 #include <vnet/fib/ip4_fib.h>
  51 #include <vnet/dpo/load_balance.h>
  52 #include <vnet/dpo/load_balance_map.h>
  53 #include <vnet/dpo/classify_dpo.h>
  54 #include <vnet/mfib/mfib_table.h>       /* for mFIB table and entry creation */
  55
  56 /**
  57  * @file
  58  * @brief IPv4 Forwarding.
  59  *
  60  * This file contains the source code for IPv4 forwarding.
  61  */
  62
  63 always_inline uword
  64 ip4_lookup_inline (vlib_main_t * vm,
  65                    vlib_node_runtime_t * node,
  66                    vlib_frame_t * frame,
  67                    int lookup_for_responses_to_locally_received_packets)
  68 {
  69   ip4_main_t *im = &ip4_main;
  70   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters;
  71   u32 n_left_from, n_left_to_next, *from, *to_next;
  72   ip_lookup_next_t next;
  73   u32 thread_index = vlib_get_thread_index ();
  74
  75   from = vlib_frame_vector_args (frame);
  76   n_left_from = frame->n_vectors;
  77   next = node->cached_next_index;
  78
  79   while (n_left_from > 0)
  80     {
  81       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
  82
  83       while (n_left_from >= 8 && n_left_to_next >= 4)
  84         {
  85           vlib_buffer_t *p0, *p1, *p2, *p3;
  86           ip4_header_t *ip0, *ip1, *ip2, *ip3;
  87           ip_lookup_next_t next0, next1, next2, next3;
  88           const load_balance_t *lb0, *lb1, *lb2, *lb3;
  89           ip4_fib_mtrie_t *mtrie0, *mtrie1, *mtrie2, *mtrie3;
  90           ip4_fib_mtrie_leaf_t leaf0, leaf1, leaf2, leaf3;
  91           ip4_address_t *dst_addr0, *dst_addr1, *dst_addr2, *dst_addr3;
  92           u32 pi0, fib_index0, lb_index0;
  93           u32 pi1, fib_index1, lb_index1;
  94           u32 pi2, fib_index2, lb_index2;
  95           u32 pi3, fib_index3, lb_index3;
  96           flow_hash_config_t flow_hash_config0, flow_hash_config1;
  97           flow_hash_config_t flow_hash_config2, flow_hash_config3;
  98           u32 hash_c0, hash_c1, hash_c2, hash_c3;
  99           const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3;
 100
 101           /* Prefetch next iteration. */
 102           {
 103             vlib_buffer_t *p4, *p5, *p6, *p7;
 104
 105             p4 = vlib_get_buffer (vm, from[4]);
 106             p5 = vlib_get_buffer (vm, from[5]);
 107             p6 = vlib_get_buffer (vm, from[6]);
 108             p7 = vlib_get_buffer (vm, from[7]);
 109
 110             vlib_prefetch_buffer_header (p4, LOAD);
 111             vlib_prefetch_buffer_header (p5, LOAD);
 112             vlib_prefetch_buffer_header (p6, LOAD);
 113             vlib_prefetch_buffer_header (p7, LOAD);
 114
 115             CLIB_PREFETCH (p4->data, sizeof (ip0[0]), LOAD);
 116             CLIB_PREFETCH (p5->data, sizeof (ip0[0]), LOAD);
 117             CLIB_PREFETCH (p6->data, sizeof (ip0[0]), LOAD);
 118             CLIB_PREFETCH (p7->data, sizeof (ip0[0]), LOAD);
 119           }
 120
 121           pi0 = to_next[0] = from[0];
 122           pi1 = to_next[1] = from[1];
 123           pi2 = to_next[2] = from[2];
 124           pi3 = to_next[3] = from[3];
 125
 126           from += 4;
 127           to_next += 4;
 128           n_left_to_next -= 4;
 129           n_left_from -= 4;
 130
 131           p0 = vlib_get_buffer (vm, pi0);
 132           p1 = vlib_get_buffer (vm, pi1);
 133           p2 = vlib_get_buffer (vm, pi2);
 134           p3 = vlib_get_buffer (vm, pi3);
 135
 136           ip0 = vlib_buffer_get_current (p0);
 137           ip1 = vlib_buffer_get_current (p1);
 138           ip2 = vlib_buffer_get_current (p2);
 139           ip3 = vlib_buffer_get_current (p3);
 140
 141           dst_addr0 = &ip0->dst_address;
 142           dst_addr1 = &ip1->dst_address;
 143           dst_addr2 = &ip2->dst_address;
 144           dst_addr3 = &ip3->dst_address;
 145
 146           fib_index0 =
 147             vec_elt (im->fib_index_by_sw_if_index,
 148                      vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 149           fib_index1 =
 150             vec_elt (im->fib_index_by_sw_if_index,
 151                      vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 152           fib_index2 =
 153             vec_elt (im->fib_index_by_sw_if_index,
 154                      vnet_buffer (p2)->sw_if_index[VLIB_RX]);
 155           fib_index3 =
 156             vec_elt (im->fib_index_by_sw_if_index,
 157                      vnet_buffer (p3)->sw_if_index[VLIB_RX]);
 158           fib_index0 =
 159             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
 160              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 161           fib_index1 =
 162             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
 163              (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
 164           fib_index2 =
 165             (vnet_buffer (p2)->sw_if_index[VLIB_TX] ==
 166              (u32) ~ 0) ? fib_index2 : vnet_buffer (p2)->sw_if_index[VLIB_TX];
 167           fib_index3 =
 168             (vnet_buffer (p3)->sw_if_index[VLIB_TX] ==
 169              (u32) ~ 0) ? fib_index3 : vnet_buffer (p3)->sw_if_index[VLIB_TX];
 170
 171
 172           if (!lookup_for_responses_to_locally_received_packets)
 173             {
 174               mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 175               mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
 176               mtrie2 = &ip4_fib_get (fib_index2)->mtrie;
 177               mtrie3 = &ip4_fib_get (fib_index3)->mtrie;
 178
 179               leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
 180               leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, dst_addr1);
 181               leaf2 = ip4_fib_mtrie_lookup_step_one (mtrie2, dst_addr2);
 182               leaf3 = ip4_fib_mtrie_lookup_step_one (mtrie3, dst_addr3);
 183             }
 184
 185           if (!lookup_for_responses_to_locally_received_packets)
 186             {
 187               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 188               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
 189               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 2);
 190               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 2);
 191             }
 192
 193           if (!lookup_for_responses_to_locally_received_packets)
 194             {
 195               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 196               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
 197               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 3);
 198               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 3);
 199             }
 200
 201           if (lookup_for_responses_to_locally_received_packets)
 202             {
 203               lb_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 204               lb_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
 205               lb_index2 = vnet_buffer (p2)->ip.adj_index[VLIB_RX];
 206               lb_index3 = vnet_buffer (p3)->ip.adj_index[VLIB_RX];
 207             }
 208           else
 209             {
 210               lb_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 211               lb_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 212               lb_index2 = ip4_fib_mtrie_leaf_get_adj_index (leaf2);
 213               lb_index3 = ip4_fib_mtrie_leaf_get_adj_index (leaf3);
 214             }
 215
 216           ASSERT (lb_index0 && lb_index1 && lb_index2 && lb_index3);
 217           lb0 = load_balance_get (lb_index0);
 218           lb1 = load_balance_get (lb_index1);
 219           lb2 = load_balance_get (lb_index2);
 220           lb3 = load_balance_get (lb_index3);
 221
 222           ASSERT (lb0->lb_n_buckets > 0);
 223           ASSERT (is_pow2 (lb0->lb_n_buckets));
 224           ASSERT (lb1->lb_n_buckets > 0);
 225           ASSERT (is_pow2 (lb1->lb_n_buckets));
 226           ASSERT (lb2->lb_n_buckets > 0);
 227           ASSERT (is_pow2 (lb2->lb_n_buckets));
 228           ASSERT (lb3->lb_n_buckets > 0);
 229           ASSERT (is_pow2 (lb3->lb_n_buckets));
 230
 231           /* Use flow hash to compute multipath adjacency. */
 232           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 233           hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
 234           hash_c2 = vnet_buffer (p2)->ip.flow_hash = 0;
 235           hash_c3 = vnet_buffer (p3)->ip.flow_hash = 0;
 236           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 237             {
 238               flow_hash_config0 = lb0->lb_hash_config;
 239               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 240                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 241               dpo0 =
 242                 load_balance_get_fwd_bucket (lb0,
 243                                              (hash_c0 &
 244                                               (lb0->lb_n_buckets_minus_1)));
 245             }
 246           else
 247             {
 248               dpo0 = load_balance_get_bucket_i (lb0, 0);
 249             }
 250           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 251             {
 252               flow_hash_config1 = lb1->lb_hash_config;
 253               hash_c1 = vnet_buffer (p1)->ip.flow_hash =
 254                 ip4_compute_flow_hash (ip1, flow_hash_config1);
 255               dpo1 =
 256                 load_balance_get_fwd_bucket (lb1,
 257                                              (hash_c1 &
 258                                               (lb1->lb_n_buckets_minus_1)));
 259             }
 260           else
 261             {
 262               dpo1 = load_balance_get_bucket_i (lb1, 0);
 263             }
 264           if (PREDICT_FALSE (lb2->lb_n_buckets > 1))
 265             {
 266               flow_hash_config2 = lb2->lb_hash_config;
 267               hash_c2 = vnet_buffer (p2)->ip.flow_hash =
 268                 ip4_compute_flow_hash (ip2, flow_hash_config2);
 269               dpo2 =
 270                 load_balance_get_fwd_bucket (lb2,
 271                                              (hash_c2 &
 272                                               (lb2->lb_n_buckets_minus_1)));
 273             }
 274           else
 275             {
 276               dpo2 = load_balance_get_bucket_i (lb2, 0);
 277             }
 278           if (PREDICT_FALSE (lb3->lb_n_buckets > 1))
 279             {
 280               flow_hash_config3 = lb3->lb_hash_config;
 281               hash_c3 = vnet_buffer (p3)->ip.flow_hash =
 282                 ip4_compute_flow_hash (ip3, flow_hash_config3);
 283               dpo3 =
 284                 load_balance_get_fwd_bucket (lb3,
 285                                              (hash_c3 &
 286                                               (lb3->lb_n_buckets_minus_1)));
 287             }
 288           else
 289             {
 290               dpo3 = load_balance_get_bucket_i (lb3, 0);
 291             }
 292
 293           next0 = dpo0->dpoi_next_node;
 294           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 295           next1 = dpo1->dpoi_next_node;
 296           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 297           next2 = dpo2->dpoi_next_node;
 298           vnet_buffer (p2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
 299           next3 = dpo3->dpoi_next_node;
 300           vnet_buffer (p3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
 301
 302           vlib_increment_combined_counter
 303             (cm, thread_index, lb_index0, 1,
 304              vlib_buffer_length_in_chain (vm, p0));
 305           vlib_increment_combined_counter
 306             (cm, thread_index, lb_index1, 1,
 307              vlib_buffer_length_in_chain (vm, p1));
 308           vlib_increment_combined_counter
 309             (cm, thread_index, lb_index2, 1,
 310              vlib_buffer_length_in_chain (vm, p2));
 311           vlib_increment_combined_counter
 312             (cm, thread_index, lb_index3, 1,
 313              vlib_buffer_length_in_chain (vm, p3));
 314
 315           vlib_validate_buffer_enqueue_x4 (vm, node, next,
 316                                            to_next, n_left_to_next,
 317                                            pi0, pi1, pi2, pi3,
 318                                            next0, next1, next2, next3);
 319         }
 320
 321       while (n_left_from > 0 && n_left_to_next > 0)
 322         {
 323           vlib_buffer_t *p0;
 324           ip4_header_t *ip0;
 325           ip_lookup_next_t next0;
 326           const load_balance_t *lb0;
 327           ip4_fib_mtrie_t *mtrie0;
 328           ip4_fib_mtrie_leaf_t leaf0;
 329           ip4_address_t *dst_addr0;
 330           u32 pi0, fib_index0, lbi0;
 331           flow_hash_config_t flow_hash_config0;
 332           const dpo_id_t *dpo0;
 333           u32 hash_c0;
 334
 335           pi0 = from[0];
 336           to_next[0] = pi0;
 337
 338           p0 = vlib_get_buffer (vm, pi0);
 339
 340           ip0 = vlib_buffer_get_current (p0);
 341
 342           dst_addr0 = &ip0->dst_address;
 343
 344           fib_index0 =
 345             vec_elt (im->fib_index_by_sw_if_index,
 346                      vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 347           fib_index0 =
 348             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
 349              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 350
 351           if (!lookup_for_responses_to_locally_received_packets)
 352             {
 353               mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 354
 355               leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
 356             }
 357
 358           if (!lookup_for_responses_to_locally_received_packets)
 359             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 360
 361           if (!lookup_for_responses_to_locally_received_packets)
 362             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 363
 364           if (lookup_for_responses_to_locally_received_packets)
 365             lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 366           else
 367             {
 368               /* Handle default route. */
 369               lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 370             }
 371
 372           ASSERT (lbi0);
 373           lb0 = load_balance_get (lbi0);
 374
 375           ASSERT (lb0->lb_n_buckets > 0);
 376           ASSERT (is_pow2 (lb0->lb_n_buckets));
 377
 378           /* Use flow hash to compute multipath adjacency. */
 379           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 380           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 381             {
 382               flow_hash_config0 = lb0->lb_hash_config;
 383
 384               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 385                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 386               dpo0 =
 387                 load_balance_get_fwd_bucket (lb0,
 388                                              (hash_c0 &
 389                                               (lb0->lb_n_buckets_minus_1)));
 390             }
 391           else
 392             {
 393               dpo0 = load_balance_get_bucket_i (lb0, 0);
 394             }
 395
 396           next0 = dpo0->dpoi_next_node;
 397           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 398
 399           vlib_increment_combined_counter (cm, thread_index, lbi0, 1,
 400                                            vlib_buffer_length_in_chain (vm,
 401                                                                         p0));
 402
 403           from += 1;
 404           to_next += 1;
 405           n_left_to_next -= 1;
 406           n_left_from -= 1;
 407
 408           if (PREDICT_FALSE (next0 != next))
 409             {
 410               n_left_to_next += 1;
 411               vlib_put_next_frame (vm, node, next, n_left_to_next);
 412               next = next0;
 413               vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 414               to_next[0] = pi0;
 415               to_next += 1;
 416               n_left_to_next -= 1;
 417             }
 418         }
 419
 420       vlib_put_next_frame (vm, node, next, n_left_to_next);
 421     }
 422
 423   if (node->flags & VLIB_NODE_FLAG_TRACE)
 424     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 425
 426   return frame->n_vectors;
 427 }
 428
 429 /** @brief IPv4 lookup node.
 430     @node ip4-lookup
 431
 432     This is the main IPv4 lookup dispatch node.
 433
 434     @param vm vlib_main_t corresponding to the current thread
 435     @param node vlib_node_runtime_t
 436     @param frame vlib_frame_t whose contents should be dispatched
 437
 438     @par Graph mechanics: buffer metadata, next index usage
 439
 440     @em Uses:
 441     - <code>vnet_buffer(b)->sw_if_index[VLIB_RX]</code>
 442         - Indicates the @c sw_if_index value of the interface that the
 443           packet was received on.
 444     - <code>vnet_buffer(b)->sw_if_index[VLIB_TX]</code>
 445         - When the value is @c ~0 then the node performs a longest prefix
 446           match (LPM) for the packet destination address in the FIB attached
 447           to the receive interface.
 448         - Otherwise perform LPM for the packet destination address in the
 449           indicated FIB. In this case <code>[VLIB_TX]</code> is a FIB index
 450           value (0, 1, ...) and not a VRF id.
 451
 452     @em Sets:
 453     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
 454         - The lookup result adjacency index.
 455
 456     <em>Next Index:</em>
 457     - Dispatches the packet to the node index found in
 458       ip_adjacency_t @c adj->lookup_next_index
 459       (where @c adj is the lookup result adjacency).
 460 */
 461 static uword
 462 ip4_lookup (vlib_main_t * vm,
 463             vlib_node_runtime_t * node, vlib_frame_t * frame)
 464 {
 465   return ip4_lookup_inline (vm, node, frame,
 466                             /* lookup_for_responses_to_locally_received_packets */
 467                             0);
 468
 469 }
 470
 471 static u8 *format_ip4_lookup_trace (u8 * s, va_list * args);
 472
 473 VLIB_REGISTER_NODE (ip4_lookup_node) =
 474 {
 475 .function = ip4_lookup,.name = "ip4-lookup",.vector_size =
 476     sizeof (u32),.format_trace = format_ip4_lookup_trace,.n_next_nodes =
 477     IP_LOOKUP_N_NEXT,.next_nodes = IP4_LOOKUP_NEXT_NODES,};
 478
 479 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup);
 480
 481 always_inline uword
 482 ip4_load_balance (vlib_main_t * vm,
 483                   vlib_node_runtime_t * node, vlib_frame_t * frame)
 484 {
 485   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 486   u32 n_left_from, n_left_to_next, *from, *to_next;
 487   ip_lookup_next_t next;
 488   u32 thread_index = vlib_get_thread_index ();
 489
 490   from = vlib_frame_vector_args (frame);
 491   n_left_from = frame->n_vectors;
 492   next = node->cached_next_index;
 493
 494   if (node->flags & VLIB_NODE_FLAG_TRACE)
 495     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 496
 497   while (n_left_from > 0)
 498     {
 499       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 500
 501
 502       while (n_left_from >= 4 && n_left_to_next >= 2)
 503         {
 504           ip_lookup_next_t next0, next1;
 505           const load_balance_t *lb0, *lb1;
 506           vlib_buffer_t *p0, *p1;
 507           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 508           const ip4_header_t *ip0, *ip1;
 509           const dpo_id_t *dpo0, *dpo1;
 510
 511           /* Prefetch next iteration. */
 512           {
 513             vlib_buffer_t *p2, *p3;
 514
 515             p2 = vlib_get_buffer (vm, from[2]);
 516             p3 = vlib_get_buffer (vm, from[3]);
 517
 518             vlib_prefetch_buffer_header (p2, STORE);
 519             vlib_prefetch_buffer_header (p3, STORE);
 520
 521             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 522             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 523           }
 524
 525           pi0 = to_next[0] = from[0];
 526           pi1 = to_next[1] = from[1];
 527
 528           from += 2;
 529           n_left_from -= 2;
 530           to_next += 2;
 531           n_left_to_next -= 2;
 532
 533           p0 = vlib_get_buffer (vm, pi0);
 534           p1 = vlib_get_buffer (vm, pi1);
 535
 536           ip0 = vlib_buffer_get_current (p0);
 537           ip1 = vlib_buffer_get_current (p1);
 538           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 539           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 540
 541           lb0 = load_balance_get (lbi0);
 542           lb1 = load_balance_get (lbi1);
 543
 544           /*
 545            * this node is for via FIBs we can re-use the hash value from the
 546            * to node if present.
 547            * We don't want to use the same hash value at each level in the recursion
 548            * graph as that would lead to polarisation
 549            */
 550           hc0 = hc1 = 0;
 551
 552           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 553             {
 554               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 555                 {
 556                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 557                     vnet_buffer (p0)->ip.flow_hash >> 1;
 558                 }
 559               else
 560                 {
 561                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 562                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 563                 }
 564               dpo0 = load_balance_get_fwd_bucket
 565                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 566             }
 567           else
 568             {
 569               dpo0 = load_balance_get_bucket_i (lb0, 0);
 570             }
 571           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 572             {
 573               if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash))
 574                 {
 575                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 576                     vnet_buffer (p1)->ip.flow_hash >> 1;
 577                 }
 578               else
 579                 {
 580                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 581                     ip4_compute_flow_hash (ip1, lb1->lb_hash_config);
 582                 }
 583               dpo1 = load_balance_get_fwd_bucket
 584                 (lb1, (hc1 & (lb1->lb_n_buckets_minus_1)));
 585             }
 586           else
 587             {
 588               dpo1 = load_balance_get_bucket_i (lb1, 0);
 589             }
 590
 591           next0 = dpo0->dpoi_next_node;
 592           next1 = dpo1->dpoi_next_node;
 593
 594           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 595           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 596
 597           vlib_increment_combined_counter
 598             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 599           vlib_increment_combined_counter
 600             (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 601
 602           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 603                                            to_next, n_left_to_next,
 604                                            pi0, pi1, next0, next1);
 605         }
 606
 607       while (n_left_from > 0 && n_left_to_next > 0)
 608         {
 609           ip_lookup_next_t next0;
 610           const load_balance_t *lb0;
 611           vlib_buffer_t *p0;
 612           u32 pi0, lbi0, hc0;
 613           const ip4_header_t *ip0;
 614           const dpo_id_t *dpo0;
 615
 616           pi0 = from[0];
 617           to_next[0] = pi0;
 618           from += 1;
 619           to_next += 1;
 620           n_left_to_next -= 1;
 621           n_left_from -= 1;
 622
 623           p0 = vlib_get_buffer (vm, pi0);
 624
 625           ip0 = vlib_buffer_get_current (p0);
 626           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 627
 628           lb0 = load_balance_get (lbi0);
 629
 630           hc0 = 0;
 631           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 632             {
 633               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 634                 {
 635                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 636                     vnet_buffer (p0)->ip.flow_hash >> 1;
 637                 }
 638               else
 639                 {
 640                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 641                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 642                 }
 643               dpo0 = load_balance_get_fwd_bucket
 644                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 645             }
 646           else
 647             {
 648               dpo0 = load_balance_get_bucket_i (lb0, 0);
 649             }
 650
 651           next0 = dpo0->dpoi_next_node;
 652           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 653
 654           vlib_increment_combined_counter
 655             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 656
 657           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 658                                            to_next, n_left_to_next,
 659                                            pi0, next0);
 660         }
 661
 662       vlib_put_next_frame (vm, node, next, n_left_to_next);
 663     }
 664
 665   return frame->n_vectors;
 666 }
 667
 668 VLIB_REGISTER_NODE (ip4_load_balance_node) =
 669 {
 670 .function = ip4_load_balance,.name = "ip4-load-balance",.vector_size =
 671     sizeof (u32),.sibling_of = "ip4-lookup",.format_trace =
 672     format_ip4_lookup_trace,};
 673
 674 VLIB_NODE_FUNCTION_MULTIARCH (ip4_load_balance_node, ip4_load_balance);
 675
 676 /* get first interface address */
 677 ip4_address_t *
 678 ip4_interface_first_address (ip4_main_t * im, u32 sw_if_index,
 679                              ip_interface_address_t ** result_ia)
 680 {
 681   ip_lookup_main_t *lm = &im->lookup_main;
 682   ip_interface_address_t *ia = 0;
 683   ip4_address_t *result = 0;
 684
 685   /* *INDENT-OFF* */
 686   foreach_ip_interface_address
 687     (lm, ia, sw_if_index,
 688      1 /* honor unnumbered */ ,
 689      ({
 690        ip4_address_t * a =
 691          ip_interface_address_get_address (lm, ia);
 692        result = a;
 693        break;
 694      }));
 695   /* *INDENT-OFF* */
 696   if (result_ia)
 697     *result_ia = result ? ia : 0;
 698   return result;
 699 }
 700
 701 static void
 702 ip4_add_interface_routes (u32 sw_if_index,
 703                           ip4_main_t * im, u32 fib_index,
 704                           ip_interface_address_t * a)
 705 {
 706   ip_lookup_main_t *lm = &im->lookup_main;
 707   ip4_address_t *address = ip_interface_address_get_address (lm, a);
 708   fib_prefix_t pfx = {
 709     .fp_len = a->address_length,
 710     .fp_proto = FIB_PROTOCOL_IP4,
 711     .fp_addr.ip4 = *address,
 712   };
 713
 714   if (pfx.fp_len <= 30)
 715     {
 716       /* a /30 or shorter - add a glean for the network address */
 717       fib_table_entry_update_one_path (fib_index, &pfx,
 718                                        FIB_SOURCE_INTERFACE,
 719                                        (FIB_ENTRY_FLAG_CONNECTED |
 720                                         FIB_ENTRY_FLAG_ATTACHED),
 721                                        DPO_PROTO_IP4,
 722                                        /* No next-hop address */
 723                                        NULL,
 724                                        sw_if_index,
 725                                        // invalid FIB index
 726                                        ~0,
 727                                        1,
 728                                        // no out-label stack
 729                                        NULL,
 730                                        FIB_ROUTE_PATH_FLAG_NONE);
 731
 732       /* Add the two broadcast addresses as drop */
 733       fib_prefix_t net_pfx = {
 734         .fp_len = 32,
 735         .fp_proto = FIB_PROTOCOL_IP4,
 736         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 737       };
 738       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 739         fib_table_entry_special_add(fib_index,
 740                                     &net_pfx,
 741                                     FIB_SOURCE_INTERFACE,
 742                                     (FIB_ENTRY_FLAG_DROP |
 743                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 744       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 745       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 746         fib_table_entry_special_add(fib_index,
 747                                     &net_pfx,
 748                                     FIB_SOURCE_INTERFACE,
 749                                     (FIB_ENTRY_FLAG_DROP |
 750                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 751     }
 752   else if (pfx.fp_len == 31)
 753     {
 754       u32 mask = clib_host_to_net_u32(1);
 755       fib_prefix_t net_pfx = pfx;
 756
 757       net_pfx.fp_len = 32;
 758       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 759
 760       /* a /31 - add the other end as an attached host */
 761       fib_table_entry_update_one_path (fib_index, &net_pfx,
 762                                        FIB_SOURCE_INTERFACE,
 763                                        (FIB_ENTRY_FLAG_ATTACHED),
 764                                        DPO_PROTO_IP4,
 765                                        &net_pfx.fp_addr,
 766                                        sw_if_index,
 767                                        // invalid FIB index
 768                                        ~0,
 769                                        1,
 770                                        NULL,
 771                                        FIB_ROUTE_PATH_FLAG_NONE);
 772     }
 773   pfx.fp_len = 32;
 774
 775   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 776     {
 777       u32 classify_table_index =
 778         lm->classify_table_index_by_sw_if_index[sw_if_index];
 779       if (classify_table_index != (u32) ~ 0)
 780         {
 781           dpo_id_t dpo = DPO_INVALID;
 782
 783           dpo_set (&dpo,
 784                    DPO_CLASSIFY,
 785                    DPO_PROTO_IP4,
 786                    classify_dpo_create (DPO_PROTO_IP4, classify_table_index));
 787
 788           fib_table_entry_special_dpo_add (fib_index,
 789                                            &pfx,
 790                                            FIB_SOURCE_CLASSIFY,
 791                                            FIB_ENTRY_FLAG_NONE, &dpo);
 792           dpo_reset (&dpo);
 793         }
 794     }
 795
 796   fib_table_entry_update_one_path (fib_index, &pfx,
 797                                    FIB_SOURCE_INTERFACE,
 798                                    (FIB_ENTRY_FLAG_CONNECTED |
 799                                     FIB_ENTRY_FLAG_LOCAL),
 800                                    DPO_PROTO_IP4,
 801                                    &pfx.fp_addr,
 802                                    sw_if_index,
 803                                    // invalid FIB index
 804                                    ~0,
 805                                    1, NULL,
 806                                    FIB_ROUTE_PATH_FLAG_NONE);
 807 }
 808
 809 static void
 810 ip4_del_interface_routes (ip4_main_t * im,
 811                           u32 fib_index,
 812                           ip4_address_t * address, u32 address_length)
 813 {
 814   fib_prefix_t pfx = {
 815     .fp_len = address_length,
 816     .fp_proto = FIB_PROTOCOL_IP4,
 817     .fp_addr.ip4 = *address,
 818   };
 819
 820   if (pfx.fp_len <= 30)
 821     {
 822       fib_prefix_t net_pfx = {
 823         .fp_len = 32,
 824         .fp_proto = FIB_PROTOCOL_IP4,
 825         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 826       };
 827       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 828         fib_table_entry_special_remove(fib_index,
 829                                        &net_pfx,
 830                                        FIB_SOURCE_INTERFACE);
 831       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 832       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 833         fib_table_entry_special_remove(fib_index,
 834                                        &net_pfx,
 835                                        FIB_SOURCE_INTERFACE);
 836       fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 837     }
 838     else if (pfx.fp_len == 31)
 839     {
 840       u32 mask = clib_host_to_net_u32(1);
 841       fib_prefix_t net_pfx = pfx;
 842
 843       net_pfx.fp_len = 32;
 844       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 845
 846       fib_table_entry_delete (fib_index, &net_pfx, FIB_SOURCE_INTERFACE);
 847     }
 848
 849   pfx.fp_len = 32;
 850   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 851 }
 852
 853 void
 854 ip4_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 855 {
 856   ip4_main_t *im = &ip4_main;
 857
 858   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 859
 860   /*
 861    * enable/disable only on the 1<->0 transition
 862    */
 863   if (is_enable)
 864     {
 865       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 866         return;
 867     }
 868   else
 869     {
 870       ASSERT (im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 871       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 872         return;
 873     }
 874   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
 875                                !is_enable, 0, 0);
 876
 877
 878   vnet_feature_enable_disable ("ip4-multicast", "ip4-drop",
 879                                sw_if_index, !is_enable, 0, 0);
 880 }
 881
 882 static clib_error_t *
 883 ip4_add_del_interface_address_internal (vlib_main_t * vm,
 884                                         u32 sw_if_index,
 885                                         ip4_address_t * address,
 886                                         u32 address_length, u32 is_del)
 887 {
 888   vnet_main_t *vnm = vnet_get_main ();
 889   ip4_main_t *im = &ip4_main;
 890   ip_lookup_main_t *lm = &im->lookup_main;
 891   clib_error_t *error = 0;
 892   u32 if_address_index, elts_before;
 893   ip4_address_fib_t ip4_af, *addr_fib = 0;
 894
 895   /* local0 interface doesn't support IP addressing  */
 896   if (sw_if_index == 0)
 897     {
 898       return
 899        clib_error_create ("local0 interface doesn't support IP addressing");
 900     }
 901
 902   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 903   ip4_addr_fib_init (&ip4_af, address,
 904                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 905   vec_add1 (addr_fib, ip4_af);
 906
 907   /* FIXME-LATER
 908    * there is no support for adj-fib handling in the presence of overlapping
 909    * subnets on interfaces. Easy fix - disallow overlapping subnets, like
 910    * most routers do.
 911    */
 912   /* *INDENT-OFF* */
 913   if (!is_del)
 914     {
 915       /* When adding an address check that it does not conflict
 916          with an existing address. */
 917       ip_interface_address_t *ia;
 918       foreach_ip_interface_address
 919         (&im->lookup_main, ia, sw_if_index,
 920          0 /* honor unnumbered */ ,
 921          ({
 922            ip4_address_t * x =
 923              ip_interface_address_get_address
 924              (&im->lookup_main, ia);
 925            if (ip4_destination_matches_route
 926                (im, address, x, ia->address_length) ||
 927                ip4_destination_matches_route (im,
 928                                               x,
 929                                               address,
 930                                               address_length))
 931              return
 932                clib_error_create
 933                ("failed to add %U which conflicts with %U for interface %U",
 934                 format_ip4_address_and_length, address,
 935                 address_length,
 936                 format_ip4_address_and_length, x,
 937                 ia->address_length,
 938                 format_vnet_sw_if_index_name, vnm,
 939                 sw_if_index);
 940          }));
 941     }
 942   /* *INDENT-ON* */
 943
 944   elts_before = pool_elts (lm->if_address_pool);
 945
 946   error = ip_interface_address_add_del
 947     (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 948   if (error)
 949     goto done;
 950
 951   ip4_sw_interface_enable_disable (sw_if_index, !is_del);
 952
 953   if (is_del)
 954     ip4_del_interface_routes (im, ip4_af.fib_index, address, address_length);
 955   else
 956     ip4_add_interface_routes (sw_if_index,
 957                               im, ip4_af.fib_index,
 958                               pool_elt_at_index
 959                               (lm->if_address_pool, if_address_index));
 960
 961   /* If pool did not grow/shrink: add duplicate address. */
 962   if (elts_before != pool_elts (lm->if_address_pool))
 963     {
 964       ip4_add_del_interface_address_callback_t *cb;
 965       vec_foreach (cb, im->add_del_interface_address_callbacks)
 966         cb->function (im, cb->function_opaque, sw_if_index,
 967                       address, address_length, if_address_index, is_del);
 968     }
 969
 970 done:
 971   vec_free (addr_fib);
 972   return error;
 973 }
 974
 975 clib_error_t *
 976 ip4_add_del_interface_address (vlib_main_t * vm,
 977                                u32 sw_if_index,
 978                                ip4_address_t * address,
 979                                u32 address_length, u32 is_del)
 980 {
 981   return ip4_add_del_interface_address_internal
 982     (vm, sw_if_index, address, address_length, is_del);
 983 }
 984
 985 /* Built-in ip4 unicast rx feature path definition */
 986 /* *INDENT-OFF* */
 987 VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
 988 {
 989   .arc_name = "ip4-unicast",
 990   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 991   .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
 992 };
 993
 994 VNET_FEATURE_INIT (ip4_flow_classify, static) =
 995 {
 996   .arc_name = "ip4-unicast",
 997   .node_name = "ip4-flow-classify",
 998   .runs_before = VNET_FEATURES ("ip4-inacl"),
 999 };
1000
1001 VNET_FEATURE_INIT (ip4_inacl, static) =
1002 {
1003   .arc_name = "ip4-unicast",
1004   .node_name = "ip4-inacl",
1005   .runs_before = VNET_FEATURES ("ip4-source-check-via-rx"),
1006 };
1007
1008 VNET_FEATURE_INIT (ip4_source_check_1, static) =
1009 {
1010   .arc_name = "ip4-unicast",
1011   .node_name = "ip4-source-check-via-rx",
1012   .runs_before = VNET_FEATURES ("ip4-source-check-via-any"),
1013 };
1014
1015 VNET_FEATURE_INIT (ip4_source_check_2, static) =
1016 {
1017   .arc_name = "ip4-unicast",
1018   .node_name = "ip4-source-check-via-any",
1019   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
1020 };
1021
1022 VNET_FEATURE_INIT (ip4_source_and_port_range_check_rx, static) =
1023 {
1024   .arc_name = "ip4-unicast",
1025   .node_name = "ip4-source-and-port-range-check-rx",
1026   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
1027 };
1028
1029 VNET_FEATURE_INIT (ip4_policer_classify, static) =
1030 {
1031   .arc_name = "ip4-unicast",
1032   .node_name = "ip4-policer-classify",
1033   .runs_before = VNET_FEATURES ("ipsec-input-ip4"),
1034 };
1035
1036 VNET_FEATURE_INIT (ip4_ipsec, static) =
1037 {
1038   .arc_name = "ip4-unicast",
1039   .node_name = "ipsec-input-ip4",
1040   .runs_before = VNET_FEATURES ("vpath-input-ip4"),
1041 };
1042
1043 VNET_FEATURE_INIT (ip4_vpath, static) =
1044 {
1045   .arc_name = "ip4-unicast",
1046   .node_name = "vpath-input-ip4",
1047   .runs_before = VNET_FEATURES ("ip4-vxlan-bypass"),
1048 };
1049
1050 VNET_FEATURE_INIT (ip4_vxlan_bypass, static) =
1051 {
1052   .arc_name = "ip4-unicast",
1053   .node_name = "ip4-vxlan-bypass",
1054   .runs_before = VNET_FEATURES ("ip4-lookup"),
1055 };
1056
1057 VNET_FEATURE_INIT (ip4_drop, static) =
1058 {
1059   .arc_name = "ip4-unicast",
1060   .node_name = "ip4-drop",
1061   .runs_before = VNET_FEATURES ("ip4-lookup"),
1062 };
1063
1064 VNET_FEATURE_INIT (ip4_lookup, static) =
1065 {
1066   .arc_name = "ip4-unicast",
1067   .node_name = "ip4-lookup",
1068   .runs_before = 0,     /* not before any other features */
1069 };
1070
1071 /* Built-in ip4 multicast rx feature path definition */
1072 VNET_FEATURE_ARC_INIT (ip4_multicast, static) =
1073 {
1074   .arc_name = "ip4-multicast",
1075   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
1076   .arc_index_ptr = &ip4_main.lookup_main.mcast_feature_arc_index,
1077 };
1078
1079 VNET_FEATURE_INIT (ip4_vpath_mc, static) =
1080 {
1081   .arc_name = "ip4-multicast",
1082   .node_name = "vpath-input-ip4",
1083   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
1084 };
1085
1086 VNET_FEATURE_INIT (ip4_mc_drop, static) =
1087 {
1088   .arc_name = "ip4-multicast",
1089   .node_name = "ip4-drop",
1090   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
1091 };
1092
1093 VNET_FEATURE_INIT (ip4_lookup_mc, static) =
1094 {
1095   .arc_name = "ip4-multicast",
1096   .node_name = "ip4-mfib-forward-lookup",
1097   .runs_before = 0,     /* last feature */
1098 };
1099
1100 /* Source and port-range check ip4 tx feature path definition */
1101 VNET_FEATURE_ARC_INIT (ip4_output, static) =
1102 {
1103   .arc_name = "ip4-output",
1104   .start_nodes = VNET_FEATURES ("ip4-rewrite", "ip4-midchain"),
1105   .arc_index_ptr = &ip4_main.lookup_main.output_feature_arc_index,
1106 };
1107
1108 VNET_FEATURE_INIT (ip4_source_and_port_range_check_tx, static) =
1109 {
1110   .arc_name = "ip4-output",
1111   .node_name = "ip4-source-and-port-range-check-tx",
1112   .runs_before = VNET_FEATURES ("ipsec-output-ip4"),
1113 };
1114
1115 VNET_FEATURE_INIT (ip4_ipsec_output, static) =
1116 {
1117   .arc_name = "ip4-output",
1118   .node_name = "ipsec-output-ip4",
1119   .runs_before = VNET_FEATURES ("interface-output"),
1120 };
1121
1122 /* Built-in ip4 tx feature path definition */
1123 VNET_FEATURE_INIT (ip4_interface_output, static) =
1124 {
1125   .arc_name = "ip4-output",
1126   .node_name = "interface-output",
1127   .runs_before = 0,     /* not before any other features */
1128 };
1129 /* *INDENT-ON* */
1130
1131 static clib_error_t *
1132 ip4_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
1133 {
1134   ip4_main_t *im = &ip4_main;
1135
1136   /* Fill in lookup tables with default table (0). */
1137   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1138   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
1139
1140   if (!is_add)
1141     {
1142       ip4_main_t *im4 = &ip4_main;
1143       ip_lookup_main_t *lm4 = &im4->lookup_main;
1144       ip_interface_address_t *ia = 0;
1145       ip4_address_t *address;
1146       vlib_main_t *vm = vlib_get_main ();
1147
1148       /* *INDENT-OFF* */
1149       foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* honor unnumbered */,
1150       ({
1151         address = ip_interface_address_get_address (lm4, ia);
1152         ip4_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
1153       }));
1154       /* *INDENT-ON* */
1155     }
1156
1157   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
1158                                is_add, 0, 0);
1159
1160   vnet_feature_enable_disable ("ip4-multicast", "ip4-drop", sw_if_index,
1161                                is_add, 0, 0);
1162
1163   return /* no error */ 0;
1164 }
1165
1166 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
1167
1168 /* Global IP4 main. */
1169 ip4_main_t ip4_main;
1170
1171 clib_error_t *
1172 ip4_lookup_init (vlib_main_t * vm)
1173 {
1174   ip4_main_t *im = &ip4_main;
1175   clib_error_t *error;
1176   uword i;
1177
1178   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
1179     return error;
1180
1181   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
1182     {
1183       u32 m;
1184
1185       if (i < 32)
1186         m = pow2_mask (i) << (32 - i);
1187       else
1188         m = ~0;
1189       im->fib_masks[i] = clib_host_to_net_u32 (m);
1190     }
1191
1192   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
1193
1194   /* Create FIB with index 0 and table id of 0. */
1195   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0,
1196                                      FIB_SOURCE_DEFAULT_ROUTE);
1197   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0,
1198                                       MFIB_SOURCE_DEFAULT_ROUTE);
1199
1200   {
1201     pg_node_t *pn;
1202     pn = pg_get_node (ip4_lookup_node.index);
1203     pn->unformat_edit = unformat_pg_ip4_header;
1204   }
1205
1206   {
1207     ethernet_arp_header_t h;
1208
1209     memset (&h, 0, sizeof (h));
1210
1211     /* Set target ethernet address to all zeros. */
1212     memset (h.ip4_over_ethernet[1].ethernet, 0,
1213             sizeof (h.ip4_over_ethernet[1].ethernet));
1214
1215 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
1216 #define _8(f,v) h.f = v;
1217     _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
1218     _16 (l3_type, ETHERNET_TYPE_IP4);
1219     _8 (n_l2_address_bytes, 6);
1220     _8 (n_l3_address_bytes, 4);
1221     _16 (opcode, ETHERNET_ARP_OPCODE_request);
1222 #undef _16
1223 #undef _8
1224
1225     vlib_packet_template_init (vm, &im->ip4_arp_request_packet_template,
1226                                /* data */ &h,
1227                                sizeof (h),
1228                                /* alloc chunk size */ 8,
1229                                "ip4 arp");
1230   }
1231
1232   return error;
1233 }
1234
1235 VLIB_INIT_FUNCTION (ip4_lookup_init);
1236
1237 typedef struct
1238 {
1239   /* Adjacency taken. */
1240   u32 dpo_index;
1241   u32 flow_hash;
1242   u32 fib_index;
1243
1244   /* Packet data, possibly *after* rewrite. */
1245   u8 packet_data[64 - 1 * sizeof (u32)];
1246 }
1247 ip4_forward_next_trace_t;
1248
1249 u8 *
1250 format_ip4_forward_next_trace (u8 * s, va_list * args)
1251 {
1252   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1253   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1254   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1255   u32 indent = format_get_indent (s);
1256   s = format (s, "%U%U",
1257               format_white_space, indent,
1258               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1259   return s;
1260 }
1261
1262 static u8 *
1263 format_ip4_lookup_trace (u8 * s, va_list * args)
1264 {
1265   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1266   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1267   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1268   u32 indent = format_get_indent (s);
1269
1270   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
1271               t->fib_index, t->dpo_index, t->flow_hash);
1272   s = format (s, "\n%U%U",
1273               format_white_space, indent,
1274               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1275   return s;
1276 }
1277
1278 static u8 *
1279 format_ip4_rewrite_trace (u8 * s, va_list * args)
1280 {
1281   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1282   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1283   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1284   u32 indent = format_get_indent (s);
1285
1286   s = format (s, "tx_sw_if_index %d dpo-idx %d : %U flow hash: 0x%08x",
1287               t->fib_index, t->dpo_index, format_ip_adjacency,
1288               t->dpo_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
1289   s = format (s, "\n%U%U",
1290               format_white_space, indent,
1291               format_ip_adjacency_packet_data,
1292               t->dpo_index, t->packet_data, sizeof (t->packet_data));
1293   return s;
1294 }
1295
1296 /* Common trace function for all ip4-forward next nodes. */
1297 void
1298 ip4_forward_next_trace (vlib_main_t * vm,
1299                         vlib_node_runtime_t * node,
1300                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
1301 {
1302   u32 *from, n_left;
1303   ip4_main_t *im = &ip4_main;
1304
1305   n_left = frame->n_vectors;
1306   from = vlib_frame_vector_args (frame);
1307
1308   while (n_left >= 4)
1309     {
1310       u32 bi0, bi1;
1311       vlib_buffer_t *b0, *b1;
1312       ip4_forward_next_trace_t *t0, *t1;
1313
1314       /* Prefetch next iteration. */
1315       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1316       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1317
1318       bi0 = from[0];
1319       bi1 = from[1];
1320
1321       b0 = vlib_get_buffer (vm, bi0);
1322       b1 = vlib_get_buffer (vm, bi1);
1323
1324       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1325         {
1326           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1327           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1328           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1329           t0->fib_index =
1330             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1331              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1332             vec_elt (im->fib_index_by_sw_if_index,
1333                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1334
1335           clib_memcpy (t0->packet_data,
1336                        vlib_buffer_get_current (b0),
1337                        sizeof (t0->packet_data));
1338         }
1339       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1340         {
1341           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1342           t1->dpo_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1343           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1344           t1->fib_index =
1345             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
1346              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
1347             vec_elt (im->fib_index_by_sw_if_index,
1348                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
1349           clib_memcpy (t1->packet_data, vlib_buffer_get_current (b1),
1350                        sizeof (t1->packet_data));
1351         }
1352       from += 2;
1353       n_left -= 2;
1354     }
1355
1356   while (n_left >= 1)
1357     {
1358       u32 bi0;
1359       vlib_buffer_t *b0;
1360       ip4_forward_next_trace_t *t0;
1361
1362       bi0 = from[0];
1363
1364       b0 = vlib_get_buffer (vm, bi0);
1365
1366       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1367         {
1368           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1369           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1370           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1371           t0->fib_index =
1372             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1373              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1374             vec_elt (im->fib_index_by_sw_if_index,
1375                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1376           clib_memcpy (t0->packet_data, vlib_buffer_get_current (b0),
1377                        sizeof (t0->packet_data));
1378         }
1379       from += 1;
1380       n_left -= 1;
1381     }
1382 }
1383
1384 /* Compute TCP/UDP/ICMP4 checksum in software. */
1385 u16
1386 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1387                               ip4_header_t * ip0)
1388 {
1389   ip_csum_t sum0;
1390   u32 ip_header_length, payload_length_host_byte_order;
1391   u32 n_this_buffer, n_bytes_left, n_ip_bytes_this_buffer;
1392   u16 sum16;
1393   void *data_this_buffer;
1394
1395   /* Initialize checksum with ip header. */
1396   ip_header_length = ip4_header_bytes (ip0);
1397   payload_length_host_byte_order =
1398     clib_net_to_host_u16 (ip0->length) - ip_header_length;
1399   sum0 =
1400     clib_host_to_net_u32 (payload_length_host_byte_order +
1401                           (ip0->protocol << 16));
1402
1403   if (BITS (uword) == 32)
1404     {
1405       sum0 =
1406         ip_csum_with_carry (sum0,
1407                             clib_mem_unaligned (&ip0->src_address, u32));
1408       sum0 =
1409         ip_csum_with_carry (sum0,
1410                             clib_mem_unaligned (&ip0->dst_address, u32));
1411     }
1412   else
1413     sum0 =
1414       ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1415
1416   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1417   data_this_buffer = (void *) ip0 + ip_header_length;
1418   n_ip_bytes_this_buffer =
1419     p0->current_length - (((u8 *) ip0 - p0->data) - p0->current_data);
1420   if (n_this_buffer + ip_header_length > n_ip_bytes_this_buffer)
1421     {
1422       n_this_buffer = n_ip_bytes_this_buffer > ip_header_length ?
1423         n_ip_bytes_this_buffer - ip_header_length : 0;
1424     }
1425   while (1)
1426     {
1427       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1428       n_bytes_left -= n_this_buffer;
1429       if (n_bytes_left == 0)
1430         break;
1431
1432       ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1433       p0 = vlib_get_buffer (vm, p0->next_buffer);
1434       data_this_buffer = vlib_buffer_get_current (p0);
1435       n_this_buffer = p0->current_length;
1436     }
1437
1438   sum16 = ~ip_csum_fold (sum0);
1439
1440   return sum16;
1441 }
1442
1443 u32
1444 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1445 {
1446   ip4_header_t *ip0 = vlib_buffer_get_current (p0);
1447   udp_header_t *udp0;
1448   u16 sum16;
1449
1450   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1451           || ip0->protocol == IP_PROTOCOL_UDP);
1452
1453   udp0 = (void *) (ip0 + 1);
1454   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1455     {
1456       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1457                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
1458       return p0->flags;
1459     }
1460
1461   sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1462
1463   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1464                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
1465
1466   return p0->flags;
1467 }
1468
1469 /* *INDENT-OFF* */
1470 VNET_FEATURE_ARC_INIT (ip4_local) =
1471 {
1472   .arc_name  = "ip4-local",
1473   .start_nodes = VNET_FEATURES ("ip4-local"),
1474 };
1475 /* *INDENT-ON* */
1476
1477 static inline void
1478 ip4_local_validate_l4 (vlib_main_t * vm, vlib_buffer_t * p, ip4_header_t * ip,
1479                        u8 is_udp, u8 * error, u8 * good_tcp_udp)
1480 {
1481   u32 flags0;
1482   flags0 = ip4_tcp_udp_validate_checksum (vm, p);
1483   *good_tcp_udp = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1484   if (is_udp)
1485     {
1486       udp_header_t *udp;
1487       u32 ip_len, udp_len;
1488       i32 len_diff;
1489       udp = ip4_next_header (ip);
1490       /* Verify UDP length. */
1491       ip_len = clib_net_to_host_u16 (ip->length);
1492       udp_len = clib_net_to_host_u16 (udp->length);
1493
1494       len_diff = ip_len - udp_len;
1495       *good_tcp_udp &= len_diff >= 0;
1496       *error = len_diff < 0 ? IP4_ERROR_UDP_LENGTH : *error;
1497     }
1498 }
1499
1500 #define ip4_local_do_l4_check(is_tcp_udp, flags)                        \
1501     (is_tcp_udp && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED \
1502     || flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM \
1503     || flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
1504
1505 static inline uword
1506 ip4_local_inline (vlib_main_t * vm,
1507                   vlib_node_runtime_t * node,
1508                   vlib_frame_t * frame, int head_of_feature_arc)
1509 {
1510   ip4_main_t *im = &ip4_main;
1511   ip_lookup_main_t *lm = &im->lookup_main;
1512   ip_local_next_t next_index;
1513   u32 *from, *to_next, n_left_from, n_left_to_next;
1514   vlib_node_runtime_t *error_node =
1515     vlib_node_get_runtime (vm, ip4_input_node.index);
1516   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1517
1518   from = vlib_frame_vector_args (frame);
1519   n_left_from = frame->n_vectors;
1520   next_index = node->cached_next_index;
1521
1522   if (node->flags & VLIB_NODE_FLAG_TRACE)
1523     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1524
1525   while (n_left_from > 0)
1526     {
1527       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1528
1529       while (n_left_from >= 4 && n_left_to_next >= 2)
1530         {
1531           vlib_buffer_t *p0, *p1;
1532           ip4_header_t *ip0, *ip1;
1533           ip4_fib_mtrie_t *mtrie0, *mtrie1;
1534           ip4_fib_mtrie_leaf_t leaf0, leaf1;
1535           const dpo_id_t *dpo0, *dpo1;
1536           const load_balance_t *lb0, *lb1;
1537           u32 pi0, next0, fib_index0, lbi0;
1538           u32 pi1, next1, fib_index1, lbi1;
1539           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1540           u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1541           u32 sw_if_index0, sw_if_index1;
1542
1543           pi0 = to_next[0] = from[0];
1544           pi1 = to_next[1] = from[1];
1545           from += 2;
1546           n_left_from -= 2;
1547           to_next += 2;
1548           n_left_to_next -= 2;
1549
1550           next0 = next1 = IP_LOCAL_NEXT_DROP;
1551           error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1552
1553           p0 = vlib_get_buffer (vm, pi0);
1554           p1 = vlib_get_buffer (vm, pi1);
1555
1556           ip0 = vlib_buffer_get_current (p0);
1557           ip1 = vlib_buffer_get_current (p1);
1558
1559           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1560           vnet_buffer (p1)->l3_hdr_offset = p1->current_data;
1561
1562           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1563           sw_if_index1 = vnet_buffer (p1)->sw_if_index[VLIB_RX];
1564
1565           /* Treat IP frag packets as "experimental" protocol for now
1566              until support of IP frag reassembly is implemented */
1567           proto0 = ip4_is_fragment (ip0) ? 0xfe : ip0->protocol;
1568           proto1 = ip4_is_fragment (ip1) ? 0xfe : ip1->protocol;
1569
1570           if (head_of_feature_arc == 0)
1571             goto skip_checks;
1572
1573           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1574           is_udp1 = proto1 == IP_PROTOCOL_UDP;
1575           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1576           is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1577
1578           good_tcp_udp0 =
1579             (p0->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1580              && !(p0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1581                   || p0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) != 0;
1582           good_tcp_udp1 = (p1->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1583                            && !(p1->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1584                                 || p1->flags &
1585                                 VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) != 0;
1586
1587           if (PREDICT_FALSE (ip4_local_do_l4_check (is_tcp_udp0, p0->flags)
1588                              || ip4_local_do_l4_check (is_tcp_udp1,
1589                                                        p1->flags)))
1590             {
1591               if (is_tcp_udp0)
1592                 ip4_local_validate_l4 (vm, p0, ip0, is_udp0, &error0,
1593                                        &good_tcp_udp0);
1594               if (is_tcp_udp1)
1595                 ip4_local_validate_l4 (vm, p1, ip1, is_udp1, &error1,
1596                                        &good_tcp_udp1);
1597             }
1598
1599           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1600           error0 = (is_tcp_udp0 && !good_tcp_udp0
1601                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1602           error1 = (is_tcp_udp1 && !good_tcp_udp1
1603                     ? IP4_ERROR_TCP_CHECKSUM + is_udp1 : error1);
1604
1605           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1606           fib_index0 =
1607             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1608              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1609
1610           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index1);
1611           fib_index1 =
1612             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
1613              (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
1614
1615           /* TODO maybe move to lookup? */
1616           vnet_buffer (p0)->ip.fib_index = fib_index0;
1617           vnet_buffer (p1)->ip.fib_index = fib_index1;
1618
1619           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1620           mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
1621
1622           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1623           leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, &ip1->src_address);
1624           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1625                                              2);
1626           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address,
1627                                              2);
1628           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1629                                              3);
1630           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address,
1631                                              3);
1632
1633           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0 =
1634             ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1635           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1636
1637           vnet_buffer (p1)->ip.adj_index[VLIB_RX] = lbi1 =
1638             ip4_fib_mtrie_leaf_get_adj_index (leaf1);
1639           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = lbi1;
1640
1641           lb0 = load_balance_get (lbi0);
1642           lb1 = load_balance_get (lbi1);
1643           dpo0 = load_balance_get_bucket_i (lb0, 0);
1644           dpo1 = load_balance_get_bucket_i (lb1, 0);
1645
1646           /*
1647            * Must have a route to source otherwise we drop the packet.
1648            * ip4 broadcasts are accepted, e.g. to make dhcp client work
1649            *
1650            * The checks are:
1651            *  - the source is a recieve => it's from us => bogus, do this
1652            *    first since it sets a different error code.
1653            *  - uRPF check for any route to source - accept if passes.
1654            *  - allow packets destined to the broadcast address from unknown sources
1655            */
1656           if (p0->flags & VNET_BUFFER_F_IS_NATED)
1657             goto skip_check0;
1658
1659           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1660                      dpo0->dpoi_type == DPO_RECEIVE) ?
1661                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1662           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1663                      !fib_urpf_check_size (lb0->lb_urpf) &&
1664                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1665                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1666
1667         skip_check0:
1668           if (p1->flags & VNET_BUFFER_F_IS_NATED)
1669             goto skip_checks;
1670
1671           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1672                      dpo1->dpoi_type == DPO_RECEIVE) ?
1673                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error1);
1674           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1675                      !fib_urpf_check_size (lb1->lb_urpf) &&
1676                      ip1->dst_address.as_u32 != 0xFFFFFFFF)
1677                     ? IP4_ERROR_SRC_LOOKUP_MISS : error1);
1678
1679         skip_checks:
1680
1681           next0 = lm->local_next_by_ip_protocol[proto0];
1682           next1 = lm->local_next_by_ip_protocol[proto1];
1683
1684           next0 =
1685             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1686           next1 =
1687             error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1688
1689           p0->error = error0 ? error_node->errors[error0] : 0;
1690           p1->error = error1 ? error_node->errors[error1] : 0;
1691
1692           if (head_of_feature_arc)
1693             {
1694               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1695                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1696               if (PREDICT_TRUE (error1 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1697                 vnet_feature_arc_start (arc_index, sw_if_index1, &next1, p1);
1698             }
1699
1700           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1701                                            n_left_to_next, pi0, pi1,
1702                                            next0, next1);
1703         }
1704
1705       while (n_left_from > 0 && n_left_to_next > 0)
1706         {
1707           vlib_buffer_t *p0;
1708           ip4_header_t *ip0;
1709           ip4_fib_mtrie_t *mtrie0;
1710           ip4_fib_mtrie_leaf_t leaf0;
1711           u32 pi0, next0, fib_index0, lbi0;
1712           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1713           load_balance_t *lb0;
1714           const dpo_id_t *dpo0;
1715           u32 sw_if_index0;
1716
1717           pi0 = to_next[0] = from[0];
1718           from += 1;
1719           n_left_from -= 1;
1720           to_next += 1;
1721           n_left_to_next -= 1;
1722
1723           next0 = IP_LOCAL_NEXT_DROP;
1724           error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
1725
1726           p0 = vlib_get_buffer (vm, pi0);
1727           ip0 = vlib_buffer_get_current (p0);
1728           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1729           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1730
1731           /* Treat IP frag packets as "experimental" protocol for now
1732              until support of IP frag reassembly is implemented */
1733           proto0 = ip4_is_fragment (ip0) ? 0xfe : ip0->protocol;
1734
1735           if (head_of_feature_arc == 0 || p0->flags & VNET_BUFFER_F_IS_NATED)
1736             goto skip_check;
1737
1738           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1739           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1740
1741           good_tcp_udp0 =
1742             (p0->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1743              && !(p0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1744                   || p0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) != 0;
1745
1746           if (PREDICT_FALSE (ip4_local_do_l4_check (is_tcp_udp0, p0->flags)))
1747             {
1748               ip4_local_validate_l4 (vm, p0, ip0, is_udp0, &error0,
1749                                      &good_tcp_udp0);
1750             }
1751
1752           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1753           error0 = (is_tcp_udp0 && !good_tcp_udp0
1754                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1755
1756           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1757           fib_index0 =
1758             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1759              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1760           vnet_buffer (p0)->ip.fib_index = fib_index0;
1761           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1762           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1763           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1764                                              2);
1765           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1766                                              3);
1767           lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1768           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1769           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0;
1770
1771           lb0 = load_balance_get (lbi0);
1772           dpo0 = load_balance_get_bucket_i (lb0, 0);
1773
1774           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1775                      dpo0->dpoi_type == DPO_RECEIVE) ?
1776                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1777           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1778                      !fib_urpf_check_size (lb0->lb_urpf) &&
1779                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1780                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1781
1782         skip_check:
1783           next0 = lm->local_next_by_ip_protocol[proto0];
1784           next0 =
1785             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1786
1787           p0->error = error0 ? error_node->errors[error0] : 0;
1788
1789           if (head_of_feature_arc)
1790             {
1791               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1792                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1793             }
1794
1795           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1796                                            n_left_to_next, pi0, next0);
1797         }
1798       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1799     }
1800
1801   return frame->n_vectors;
1802 }
1803
1804 static uword
1805 ip4_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1806 {
1807   return ip4_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1808 }
1809
1810 /* *INDENT-OFF* */
1811 VLIB_REGISTER_NODE (ip4_local_node) =
1812 {
1813   .function = ip4_local,
1814   .name = "ip4-local",
1815   .vector_size = sizeof (u32),
1816   .format_trace = format_ip4_forward_next_trace,
1817   .n_next_nodes = IP_LOCAL_N_NEXT,
1818   .next_nodes =
1819   {
1820     [IP_LOCAL_NEXT_DROP] = "ip4-drop",
1821     [IP_LOCAL_NEXT_PUNT] = "ip4-punt",
1822     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
1823     [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",
1824   },
1825 };
1826 /* *INDENT-ON* */
1827
1828 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local);
1829
1830 static uword
1831 ip4_local_end_of_arc (vlib_main_t * vm,
1832                       vlib_node_runtime_t * node, vlib_frame_t * frame)
1833 {
1834   return ip4_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1835 }
1836
1837 /* *INDENT-OFF* */
1838 VLIB_REGISTER_NODE (ip4_local_end_of_arc_node,static) = {
1839   .function = ip4_local_end_of_arc,
1840   .name = "ip4-local-end-of-arc",
1841   .vector_size = sizeof (u32),
1842
1843   .format_trace = format_ip4_forward_next_trace,
1844   .sibling_of = "ip4-local",
1845 };
1846
1847 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_end_of_arc_node, ip4_local_end_of_arc)
1848
1849 VNET_FEATURE_INIT (ip4_local_end_of_arc, static) = {
1850   .arc_name = "ip4-local",
1851   .node_name = "ip4-local-end-of-arc",
1852   .runs_before = 0, /* not before any other features */
1853 };
1854 /* *INDENT-ON* */
1855
1856 void
1857 ip4_register_protocol (u32 protocol, u32 node_index)
1858 {
1859   vlib_main_t *vm = vlib_get_main ();
1860   ip4_main_t *im = &ip4_main;
1861   ip_lookup_main_t *lm = &im->lookup_main;
1862
1863   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1864   lm->local_next_by_ip_protocol[protocol] =
1865     vlib_node_add_next (vm, ip4_local_node.index, node_index);
1866 }
1867
1868 static clib_error_t *
1869 show_ip_local_command_fn (vlib_main_t * vm,
1870                           unformat_input_t * input, vlib_cli_command_t * cmd)
1871 {
1872   ip4_main_t *im = &ip4_main;
1873   ip_lookup_main_t *lm = &im->lookup_main;
1874   int i;
1875
1876   vlib_cli_output (vm, "Protocols handled by ip4_local");
1877   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
1878     {
1879       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
1880         {
1881           u32 node_index = vlib_get_node (vm,
1882                                           ip4_local_node.index)->
1883             next_nodes[lm->local_next_by_ip_protocol[i]];
1884           vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm,
1885                            node_index);
1886         }
1887     }
1888   return 0;
1889 }
1890
1891
1892
1893 /*?
1894  * Display the set of protocols handled by the local IPv4 stack.
1895  *
1896  * @cliexpar
1897  * Example of how to display local protocol table:
1898  * @cliexstart{show ip local}
1899  * Protocols handled by ip4_local
1900  * 1
1901  * 17
1902  * 47
1903  * @cliexend
1904 ?*/
1905 /* *INDENT-OFF* */
1906 VLIB_CLI_COMMAND (show_ip_local, static) =
1907 {
1908   .path = "show ip local",
1909   .function = show_ip_local_command_fn,
1910   .short_help = "show ip local",
1911 };
1912 /* *INDENT-ON* */
1913
1914 always_inline uword
1915 ip4_arp_inline (vlib_main_t * vm,
1916                 vlib_node_runtime_t * node,
1917                 vlib_frame_t * frame, int is_glean)
1918 {
1919   vnet_main_t *vnm = vnet_get_main ();
1920   ip4_main_t *im = &ip4_main;
1921   ip_lookup_main_t *lm = &im->lookup_main;
1922   u32 *from, *to_next_drop;
1923   uword n_left_from, n_left_to_next_drop, next_index;
1924   static f64 time_last_seed_change = -1e100;
1925   static u32 hash_seeds[3];
1926   static uword hash_bitmap[256 / BITS (uword)];
1927   f64 time_now;
1928
1929   if (node->flags & VLIB_NODE_FLAG_TRACE)
1930     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1931
1932   time_now = vlib_time_now (vm);
1933   if (time_now - time_last_seed_change > 1e-3)
1934     {
1935       uword i;
1936       u32 *r = clib_random_buffer_get_data (&vm->random_buffer,
1937                                             sizeof (hash_seeds));
1938       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1939         hash_seeds[i] = r[i];
1940
1941       /* Mark all hash keys as been no-seen before. */
1942       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1943         hash_bitmap[i] = 0;
1944
1945       time_last_seed_change = time_now;
1946     }
1947
1948   from = vlib_frame_vector_args (frame);
1949   n_left_from = frame->n_vectors;
1950   next_index = node->cached_next_index;
1951   if (next_index == IP4_ARP_NEXT_DROP)
1952     next_index = IP4_ARP_N_NEXT;        /* point to first interface */
1953
1954   while (n_left_from > 0)
1955     {
1956       vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
1957                            to_next_drop, n_left_to_next_drop);
1958
1959       while (n_left_from > 0 && n_left_to_next_drop > 0)
1960         {
1961           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
1962           ip_adjacency_t *adj0;
1963           vlib_buffer_t *p0;
1964           ip4_header_t *ip0;
1965           uword bm0;
1966
1967           pi0 = from[0];
1968
1969           p0 = vlib_get_buffer (vm, pi0);
1970
1971           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1972           adj0 = adj_get (adj_index0);
1973           ip0 = vlib_buffer_get_current (p0);
1974
1975           a0 = hash_seeds[0];
1976           b0 = hash_seeds[1];
1977           c0 = hash_seeds[2];
1978
1979           sw_if_index0 = adj0->rewrite_header.sw_if_index;
1980           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
1981
1982           if (is_glean)
1983             {
1984               /*
1985                * this is the Glean case, so we are ARPing for the
1986                * packet's destination
1987                */
1988               a0 ^= ip0->dst_address.data_u32;
1989             }
1990           else
1991             {
1992               a0 ^= adj0->sub_type.nbr.next_hop.ip4.data_u32;
1993             }
1994           b0 ^= sw_if_index0;
1995
1996           hash_v3_mix32 (a0, b0, c0);
1997           hash_v3_finalize32 (a0, b0, c0);
1998
1999           c0 &= BITS (hash_bitmap) - 1;
2000           m0 = (uword) 1 << (c0 % BITS (uword));
2001           c0 = c0 / BITS (uword);
2002
2003           bm0 = hash_bitmap[c0];
2004           drop0 = (bm0 & m0) != 0;
2005
2006           /* Mark it as seen. */
2007           hash_bitmap[c0] = bm0 | m0;
2008
2009           from += 1;
2010           n_left_from -= 1;
2011           to_next_drop[0] = pi0;
2012           to_next_drop += 1;
2013           n_left_to_next_drop -= 1;
2014
2015           p0->error =
2016             node->errors[drop0 ? IP4_ARP_ERROR_DROP :
2017                          IP4_ARP_ERROR_REQUEST_SENT];
2018
2019           /*
2020            * the adj has been updated to a rewrite but the node the DPO that got
2021            * us here hasn't - yet. no big deal. we'll drop while we wait.
2022            */
2023           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
2024             continue;
2025
2026           if (drop0)
2027             continue;
2028
2029           /*
2030            * Can happen if the control-plane is programming tables
2031            * with traffic flowing; at least that's today's lame excuse.
2032            */
2033           if ((is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_GLEAN)
2034               || (!is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP))
2035             {
2036               p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
2037             }
2038           else
2039             /* Send ARP request. */
2040             {
2041               u32 bi0 = 0;
2042               vlib_buffer_t *b0;
2043               ethernet_arp_header_t *h0;
2044               vnet_hw_interface_t *hw_if0;
2045
2046               h0 =
2047                 vlib_packet_template_get_packet (vm,
2048                                                  &im->ip4_arp_request_packet_template,
2049                                                  &bi0);
2050
2051               /* Seems we're out of buffers */
2052               if (PREDICT_FALSE (!h0))
2053                 continue;
2054
2055               /* Add rewrite/encap string for ARP packet. */
2056               vnet_rewrite_one_header (adj0[0], h0,
2057                                        sizeof (ethernet_header_t));
2058
2059               hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2060
2061               /* Src ethernet address in ARP header. */
2062               clib_memcpy (h0->ip4_over_ethernet[0].ethernet,
2063                            hw_if0->hw_address,
2064                            sizeof (h0->ip4_over_ethernet[0].ethernet));
2065
2066               if (is_glean)
2067                 {
2068                   /* The interface's source address is stashed in the Glean Adj */
2069                   h0->ip4_over_ethernet[0].ip4 =
2070                     adj0->sub_type.glean.receive_addr.ip4;
2071
2072                   /* Copy in destination address we are requesting. This is the
2073                    * glean case, so it's the packet's destination.*/
2074                   h0->ip4_over_ethernet[1].ip4.data_u32 =
2075                     ip0->dst_address.data_u32;
2076                 }
2077               else
2078                 {
2079                   /* Src IP address in ARP header. */
2080                   if (ip4_src_address_for_packet (lm, sw_if_index0,
2081                                                   &h0->
2082                                                   ip4_over_ethernet[0].ip4))
2083                     {
2084                       /* No source address available */
2085                       p0->error =
2086                         node->errors[IP4_ARP_ERROR_NO_SOURCE_ADDRESS];
2087                       vlib_buffer_free (vm, &bi0, 1);
2088                       continue;
2089                     }
2090
2091                   /* Copy in destination address we are requesting from the
2092                      incomplete adj */
2093                   h0->ip4_over_ethernet[1].ip4.data_u32 =
2094                     adj0->sub_type.nbr.next_hop.ip4.as_u32;
2095                 }
2096
2097               vlib_buffer_copy_trace_flag (vm, p0, bi0);
2098               b0 = vlib_get_buffer (vm, bi0);
2099               VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
2100               vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
2101
2102               vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2103
2104               vlib_set_next_frame_buffer (vm, node,
2105                                           adj0->rewrite_header.next_index,
2106                                           bi0);
2107             }
2108         }
2109
2110       vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
2111     }
2112
2113   return frame->n_vectors;
2114 }
2115
2116 static uword
2117 ip4_arp (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
2118 {
2119   return (ip4_arp_inline (vm, node, frame, 0));
2120 }
2121
2122 static uword
2123 ip4_glean (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
2124 {
2125   return (ip4_arp_inline (vm, node, frame, 1));
2126 }
2127
2128 static char *ip4_arp_error_strings[] = {
2129   [IP4_ARP_ERROR_DROP] = "address overflow drops",
2130   [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
2131   [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
2132   [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
2133   [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
2134   [IP4_ARP_ERROR_NO_SOURCE_ADDRESS] = "no source address for ARP request",
2135 };
2136
2137 VLIB_REGISTER_NODE (ip4_arp_node) =
2138 {
2139   .function = ip4_arp,.name = "ip4-arp",.vector_size =
2140     sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_errors =
2141     ARRAY_LEN (ip4_arp_error_strings),.error_strings =
2142     ip4_arp_error_strings,.n_next_nodes = IP4_ARP_N_NEXT,.next_nodes =
2143   {
2144   [IP4_ARP_NEXT_DROP] = "error-drop",}
2145 ,};
2146
2147 VLIB_REGISTER_NODE (ip4_glean_node) =
2148 {
2149   .function = ip4_glean,.name = "ip4-glean",.vector_size =
2150     sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_errors =
2151     ARRAY_LEN (ip4_arp_error_strings),.error_strings =
2152     ip4_arp_error_strings,.n_next_nodes = IP4_ARP_N_NEXT,.next_nodes =
2153   {
2154   [IP4_ARP_NEXT_DROP] = "error-drop",}
2155 ,};
2156
2157 #define foreach_notrace_ip4_arp_error           \
2158 _(DROP)                                         \
2159 _(REQUEST_SENT)                                 \
2160 _(REPLICATE_DROP)                               \
2161 _(REPLICATE_FAIL)
2162
2163 clib_error_t *
2164 arp_notrace_init (vlib_main_t * vm)
2165 {
2166   vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, ip4_arp_node.index);
2167
2168   /* don't trace ARP request packets */
2169 #define _(a)                                    \
2170     vnet_pcap_drop_trace_filter_add_del         \
2171         (rt->errors[IP4_ARP_ERROR_##a],         \
2172          1 /* is_add */);
2173   foreach_notrace_ip4_arp_error;
2174 #undef _
2175   return 0;
2176 }
2177
2178 VLIB_INIT_FUNCTION (arp_notrace_init);
2179
2180
2181 /* Send an ARP request to see if given destination is reachable on given interface. */
2182 clib_error_t *
2183 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
2184 {
2185   vnet_main_t *vnm = vnet_get_main ();
2186   ip4_main_t *im = &ip4_main;
2187   ethernet_arp_header_t *h;
2188   ip4_address_t *src;
2189   ip_interface_address_t *ia;
2190   ip_adjacency_t *adj;
2191   vnet_hw_interface_t *hi;
2192   vnet_sw_interface_t *si;
2193   vlib_buffer_t *b;
2194   adj_index_t ai;
2195   u32 bi = 0;
2196
2197   si = vnet_get_sw_interface (vnm, sw_if_index);
2198
2199   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2200     {
2201       return clib_error_return (0, "%U: interface %U down",
2202                                 format_ip4_address, dst,
2203                                 format_vnet_sw_if_index_name, vnm,
2204                                 sw_if_index);
2205     }
2206
2207   src =
2208     ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2209   if (!src)
2210     {
2211       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2212       return clib_error_return
2213         (0,
2214          "no matching interface address for destination %U (interface %U)",
2215          format_ip4_address, dst, format_vnet_sw_if_index_name, vnm,
2216          sw_if_index);
2217     }
2218
2219   h = vlib_packet_template_get_packet (vm,
2220                                        &im->ip4_arp_request_packet_template,
2221                                        &bi);
2222
2223   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2224   if (PREDICT_FALSE (!hi->hw_address))
2225     {
2226       return clib_error_return (0, "%U: interface %U do not support ip probe",
2227                                 format_ip4_address, dst,
2228                                 format_vnet_sw_if_index_name, vnm,
2229                                 sw_if_index);
2230     }
2231
2232   clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address,
2233                sizeof (h->ip4_over_ethernet[0].ethernet));
2234
2235   h->ip4_over_ethernet[0].ip4 = src[0];
2236   h->ip4_over_ethernet[1].ip4 = dst[0];
2237
2238   b = vlib_get_buffer (vm, bi);
2239   vnet_buffer (b)->sw_if_index[VLIB_RX] =
2240     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2241
2242   ip46_address_t nh = {
2243     .ip4 = *dst,
2244   };
2245
2246   ai = adj_nbr_add_or_lock (FIB_PROTOCOL_IP4,
2247                             VNET_LINK_IP4, &nh, sw_if_index);
2248   adj = adj_get (ai);
2249
2250   /* Peer has been previously resolved, retrieve glean adj instead */
2251   if (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE)
2252     {
2253       adj_unlock (ai);
2254       ai = adj_glean_add_or_lock (FIB_PROTOCOL_IP4, sw_if_index, &nh);
2255       adj = adj_get (ai);
2256     }
2257
2258   /* Add encapsulation string for software interface (e.g. ethernet header). */
2259   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2260   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2261
2262   {
2263     vlib_frame_t *f = vlib_get_frame_to_node (vm, hi->output_node_index);
2264     u32 *to_next = vlib_frame_vector_args (f);
2265     to_next[0] = bi;
2266     f->n_vectors = 1;
2267     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2268   }
2269
2270   adj_unlock (ai);
2271   return /* no error */ 0;
2272 }
2273
2274 typedef enum
2275 {
2276   IP4_REWRITE_NEXT_DROP,
2277   IP4_REWRITE_NEXT_ICMP_ERROR,
2278 } ip4_rewrite_next_t;
2279
2280 always_inline uword
2281 ip4_rewrite_inline (vlib_main_t * vm,
2282                     vlib_node_runtime_t * node,
2283                     vlib_frame_t * frame,
2284                     int do_counters, int is_midchain, int is_mcast)
2285 {
2286   ip_lookup_main_t *lm = &ip4_main.lookup_main;
2287   u32 *from = vlib_frame_vector_args (frame);
2288   u32 n_left_from, n_left_to_next, *to_next, next_index;
2289   vlib_node_runtime_t *error_node =
2290     vlib_node_get_runtime (vm, ip4_input_node.index);
2291
2292   n_left_from = frame->n_vectors;
2293   next_index = node->cached_next_index;
2294   u32 thread_index = vlib_get_thread_index ();
2295
2296   while (n_left_from > 0)
2297     {
2298       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2299
2300       while (n_left_from >= 4 && n_left_to_next >= 2)
2301         {
2302           ip_adjacency_t *adj0, *adj1;
2303           vlib_buffer_t *p0, *p1;
2304           ip4_header_t *ip0, *ip1;
2305           u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
2306           u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
2307           u32 tx_sw_if_index0, tx_sw_if_index1;
2308
2309           /* Prefetch next iteration. */
2310           {
2311             vlib_buffer_t *p2, *p3;
2312
2313             p2 = vlib_get_buffer (vm, from[2]);
2314             p3 = vlib_get_buffer (vm, from[3]);
2315
2316             vlib_prefetch_buffer_header (p2, STORE);
2317             vlib_prefetch_buffer_header (p3, STORE);
2318
2319             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2320             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2321           }
2322
2323           pi0 = to_next[0] = from[0];
2324           pi1 = to_next[1] = from[1];
2325
2326           from += 2;
2327           n_left_from -= 2;
2328           to_next += 2;
2329           n_left_to_next -= 2;
2330
2331           p0 = vlib_get_buffer (vm, pi0);
2332           p1 = vlib_get_buffer (vm, pi1);
2333
2334           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2335           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
2336
2337           /*
2338            * pre-fetch the per-adjacency counters
2339            */
2340           if (do_counters)
2341             {
2342               vlib_prefetch_combined_counter (&adjacency_counters,
2343                                               thread_index, adj_index0);
2344               vlib_prefetch_combined_counter (&adjacency_counters,
2345                                               thread_index, adj_index1);
2346             }
2347
2348           ip0 = vlib_buffer_get_current (p0);
2349           ip1 = vlib_buffer_get_current (p1);
2350
2351           error0 = error1 = IP4_ERROR_NONE;
2352           next0 = next1 = IP4_REWRITE_NEXT_DROP;
2353
2354           /* Decrement TTL & update checksum.
2355              Works either endian, so no need for byte swap. */
2356           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2357             {
2358               i32 ttl0 = ip0->ttl;
2359
2360               /* Input node should have reject packets with ttl 0. */
2361               ASSERT (ip0->ttl > 0);
2362
2363               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2364               checksum0 += checksum0 >= 0xffff;
2365
2366               ip0->checksum = checksum0;
2367               ttl0 -= 1;
2368               ip0->ttl = ttl0;
2369
2370               /*
2371                * If the ttl drops below 1 when forwarding, generate
2372                * an ICMP response.
2373                */
2374               if (PREDICT_FALSE (ttl0 <= 0))
2375                 {
2376                   error0 = IP4_ERROR_TIME_EXPIRED;
2377                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2378                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2379                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2380                                                0);
2381                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2382                 }
2383
2384               /* Verify checksum. */
2385               ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
2386                       (p0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2387             }
2388           else
2389             {
2390               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2391             }
2392           if (PREDICT_TRUE (!(p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2393             {
2394               i32 ttl1 = ip1->ttl;
2395
2396               /* Input node should have reject packets with ttl 0. */
2397               ASSERT (ip1->ttl > 0);
2398
2399               checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2400               checksum1 += checksum1 >= 0xffff;
2401
2402               ip1->checksum = checksum1;
2403               ttl1 -= 1;
2404               ip1->ttl = ttl1;
2405
2406               /*
2407                * If the ttl drops below 1 when forwarding, generate
2408                * an ICMP response.
2409                */
2410               if (PREDICT_FALSE (ttl1 <= 0))
2411                 {
2412                   error1 = IP4_ERROR_TIME_EXPIRED;
2413                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2414                   icmp4_error_set_vnet_buffer (p1, ICMP4_time_exceeded,
2415                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2416                                                0);
2417                   next1 = IP4_REWRITE_NEXT_ICMP_ERROR;
2418                 }
2419
2420               /* Verify checksum. */
2421               ASSERT ((ip1->checksum == ip4_header_checksum (ip1)) ||
2422                       (p1->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2423             }
2424           else
2425             {
2426               p1->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2427             }
2428
2429           /* Rewrite packet header and updates lengths. */
2430           adj0 = adj_get (adj_index0);
2431           adj1 = adj_get (adj_index1);
2432
2433           /* Worth pipelining. No guarantee that adj0,1 are hot... */
2434           rw_len0 = adj0[0].rewrite_header.data_bytes;
2435           rw_len1 = adj1[0].rewrite_header.data_bytes;
2436           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2437           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
2438
2439           /* Check MTU of outgoing interface. */
2440           error0 =
2441             (vlib_buffer_length_in_chain (vm, p0) >
2442              adj0[0].
2443              rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
2444              error0);
2445           error1 =
2446             (vlib_buffer_length_in_chain (vm, p1) >
2447              adj1[0].
2448              rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
2449              error1);
2450
2451           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2452            * to see the IP headerr */
2453           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2454             {
2455               next0 = adj0[0].rewrite_header.next_index;
2456               p0->current_data -= rw_len0;
2457               p0->current_length += rw_len0;
2458               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2459               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2460
2461               if (PREDICT_FALSE
2462                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2463                 vnet_feature_arc_start (lm->output_feature_arc_index,
2464                                         tx_sw_if_index0, &next0, p0);
2465             }
2466           if (PREDICT_TRUE (error1 == IP4_ERROR_NONE))
2467             {
2468               next1 = adj1[0].rewrite_header.next_index;
2469               p1->current_data -= rw_len1;
2470               p1->current_length += rw_len1;
2471
2472               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
2473               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
2474
2475               if (PREDICT_FALSE
2476                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2477                 vnet_feature_arc_start (lm->output_feature_arc_index,
2478                                         tx_sw_if_index1, &next1, p1);
2479             }
2480
2481           /* Guess we are only writing on simple Ethernet header. */
2482           vnet_rewrite_two_headers (adj0[0], adj1[0],
2483                                     ip0, ip1, sizeof (ethernet_header_t));
2484
2485           /*
2486            * Bump the per-adjacency counters
2487            */
2488           if (do_counters)
2489             {
2490               vlib_increment_combined_counter
2491                 (&adjacency_counters,
2492                  thread_index,
2493                  adj_index0, 1,
2494                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2495
2496               vlib_increment_combined_counter
2497                 (&adjacency_counters,
2498                  thread_index,
2499                  adj_index1, 1,
2500                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
2501             }
2502
2503           if (is_midchain)
2504             {
2505               adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
2506               adj1->sub_type.midchain.fixup_func (vm, adj1, p1);
2507             }
2508           if (is_mcast)
2509             {
2510               /*
2511                * copy bytes from the IP address into the MAC rewrite
2512                */
2513               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2514               vnet_fixup_one_header (adj1[0], &ip1->dst_address, ip1);
2515             }
2516
2517           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2518                                            to_next, n_left_to_next,
2519                                            pi0, pi1, next0, next1);
2520         }
2521
2522       while (n_left_from > 0 && n_left_to_next > 0)
2523         {
2524           ip_adjacency_t *adj0;
2525           vlib_buffer_t *p0;
2526           ip4_header_t *ip0;
2527           u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2528           u32 tx_sw_if_index0;
2529
2530           pi0 = to_next[0] = from[0];
2531
2532           p0 = vlib_get_buffer (vm, pi0);
2533
2534           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2535
2536           adj0 = adj_get (adj_index0);
2537
2538           ip0 = vlib_buffer_get_current (p0);
2539
2540           error0 = IP4_ERROR_NONE;
2541           next0 = IP4_REWRITE_NEXT_DROP;        /* drop on error */
2542
2543           /* Decrement TTL & update checksum. */
2544           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2545             {
2546               i32 ttl0 = ip0->ttl;
2547
2548               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2549
2550               checksum0 += checksum0 >= 0xffff;
2551
2552               ip0->checksum = checksum0;
2553
2554               ASSERT (ip0->ttl > 0);
2555
2556               ttl0 -= 1;
2557
2558               ip0->ttl = ttl0;
2559
2560               ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
2561                       (p0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2562
2563               if (PREDICT_FALSE (ttl0 <= 0))
2564                 {
2565                   /*
2566                    * If the ttl drops below 1 when forwarding, generate
2567                    * an ICMP response.
2568                    */
2569                   error0 = IP4_ERROR_TIME_EXPIRED;
2570                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2571                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2572                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2573                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2574                                                0);
2575                 }
2576             }
2577           else
2578             {
2579               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2580             }
2581
2582           if (do_counters)
2583             vlib_prefetch_combined_counter (&adjacency_counters,
2584                                             thread_index, adj_index0);
2585
2586           /* Guess we are only writing on simple Ethernet header. */
2587           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2588           if (is_mcast)
2589             {
2590               /*
2591                * copy bytes from the IP address into the MAC rewrite
2592                */
2593               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2594             }
2595
2596           /* Update packet buffer attributes/set output interface. */
2597           rw_len0 = adj0[0].rewrite_header.data_bytes;
2598           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2599
2600           if (do_counters)
2601             vlib_increment_combined_counter
2602               (&adjacency_counters,
2603                thread_index, adj_index0, 1,
2604                vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2605
2606           /* Check MTU of outgoing interface. */
2607           error0 = (vlib_buffer_length_in_chain (vm, p0)
2608                     > adj0[0].rewrite_header.max_l3_packet_bytes
2609                     ? IP4_ERROR_MTU_EXCEEDED : error0);
2610
2611           p0->error = error_node->errors[error0];
2612
2613           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2614            * to see the IP headerr */
2615           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2616             {
2617               p0->current_data -= rw_len0;
2618               p0->current_length += rw_len0;
2619               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2620
2621               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2622               next0 = adj0[0].rewrite_header.next_index;
2623
2624               if (is_midchain)
2625                 {
2626                   adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
2627                 }
2628
2629               if (PREDICT_FALSE
2630                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2631                 vnet_feature_arc_start (lm->output_feature_arc_index,
2632                                         tx_sw_if_index0, &next0, p0);
2633
2634             }
2635
2636           from += 1;
2637           n_left_from -= 1;
2638           to_next += 1;
2639           n_left_to_next -= 1;
2640
2641           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2642                                            to_next, n_left_to_next,
2643                                            pi0, next0);
2644         }
2645
2646       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2647     }
2648
2649   /* Need to do trace after rewrites to pick up new packet data. */
2650   if (node->flags & VLIB_NODE_FLAG_TRACE)
2651     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2652
2653   return frame->n_vectors;
2654 }
2655
2656
2657 /** @brief IPv4 rewrite node.
2658     @node ip4-rewrite
2659
2660     This is the IPv4 transit-rewrite node: decrement TTL, fix the ipv4
2661     header checksum, fetch the ip adjacency, check the outbound mtu,
2662     apply the adjacency rewrite, and send pkts to the adjacency
2663     rewrite header's rewrite_next_index.
2664
2665     @param vm vlib_main_t corresponding to the current thread
2666     @param node vlib_node_runtime_t
2667     @param frame vlib_frame_t whose contents should be dispatched
2668
2669     @par Graph mechanics: buffer metadata, next index usage
2670
2671     @em Uses:
2672     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
2673         - the rewrite adjacency index
2674     - <code>adj->lookup_next_index</code>
2675         - Must be IP_LOOKUP_NEXT_REWRITE or IP_LOOKUP_NEXT_ARP, otherwise
2676           the packet will be dropped.
2677     - <code>adj->rewrite_header</code>
2678         - Rewrite string length, rewrite string, next_index
2679
2680     @em Sets:
2681     - <code>b->current_data, b->current_length</code>
2682         - Updated net of applying the rewrite string
2683
2684     <em>Next Indices:</em>
2685     - <code> adj->rewrite_header.next_index </code>
2686       or @c error-drop
2687 */
2688 static uword
2689 ip4_rewrite (vlib_main_t * vm,
2690              vlib_node_runtime_t * node, vlib_frame_t * frame)
2691 {
2692   if (adj_are_counters_enabled ())
2693     return ip4_rewrite_inline (vm, node, frame, 1, 0, 0);
2694   else
2695     return ip4_rewrite_inline (vm, node, frame, 0, 0, 0);
2696 }
2697
2698 static uword
2699 ip4_midchain (vlib_main_t * vm,
2700               vlib_node_runtime_t * node, vlib_frame_t * frame)
2701 {
2702   if (adj_are_counters_enabled ())
2703     return ip4_rewrite_inline (vm, node, frame, 1, 1, 0);
2704   else
2705     return ip4_rewrite_inline (vm, node, frame, 0, 1, 0);
2706 }
2707
2708 static uword
2709 ip4_rewrite_mcast (vlib_main_t * vm,
2710                    vlib_node_runtime_t * node, vlib_frame_t * frame)
2711 {
2712   if (adj_are_counters_enabled ())
2713     return ip4_rewrite_inline (vm, node, frame, 1, 0, 1);
2714   else
2715     return ip4_rewrite_inline (vm, node, frame, 0, 0, 1);
2716 }
2717
2718 static uword
2719 ip4_mcast_midchain (vlib_main_t * vm,
2720                     vlib_node_runtime_t * node, vlib_frame_t * frame)
2721 {
2722   if (adj_are_counters_enabled ())
2723     return ip4_rewrite_inline (vm, node, frame, 1, 1, 1);
2724   else
2725     return ip4_rewrite_inline (vm, node, frame, 0, 1, 1);
2726 }
2727
2728 /* *INDENT-OFF* */
2729 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2730   .function = ip4_rewrite,
2731   .name = "ip4-rewrite",
2732   .vector_size = sizeof (u32),
2733
2734   .format_trace = format_ip4_rewrite_trace,
2735
2736   .n_next_nodes = 2,
2737   .next_nodes = {
2738     [IP4_REWRITE_NEXT_DROP] = "error-drop",
2739     [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2740   },
2741 };
2742 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite)
2743
2744 VLIB_REGISTER_NODE (ip4_rewrite_mcast_node) = {
2745   .function = ip4_rewrite_mcast,
2746   .name = "ip4-rewrite-mcast",
2747   .vector_size = sizeof (u32),
2748
2749   .format_trace = format_ip4_rewrite_trace,
2750   .sibling_of = "ip4-rewrite",
2751 };
2752 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_mcast_node, ip4_rewrite_mcast)
2753
2754 VLIB_REGISTER_NODE (ip4_mcast_midchain_node, static) = {
2755   .function = ip4_mcast_midchain,
2756   .name = "ip4-mcast-midchain",
2757   .vector_size = sizeof (u32),
2758
2759   .format_trace = format_ip4_rewrite_trace,
2760   .sibling_of = "ip4-rewrite",
2761 };
2762 VLIB_NODE_FUNCTION_MULTIARCH (ip4_mcast_midchain_node, ip4_mcast_midchain)
2763
2764 VLIB_REGISTER_NODE (ip4_midchain_node) = {
2765   .function = ip4_midchain,
2766   .name = "ip4-midchain",
2767   .vector_size = sizeof (u32),
2768   .format_trace = format_ip4_forward_next_trace,
2769   .sibling_of =  "ip4-rewrite",
2770 };
2771 VLIB_NODE_FUNCTION_MULTIARCH (ip4_midchain_node, ip4_midchain);
2772 /* *INDENT-ON */
2773
2774 int
2775 ip4_lookup_validate (ip4_address_t * a, u32 fib_index0)
2776 {
2777   ip4_fib_mtrie_t *mtrie0;
2778   ip4_fib_mtrie_leaf_t leaf0;
2779   u32 lbi0;
2780
2781   mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
2782
2783   leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, a);
2784   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
2785   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
2786
2787   lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2788
2789   return lbi0 == ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0), a);
2790 }
2791
2792 static clib_error_t *
2793 test_lookup_command_fn (vlib_main_t * vm,
2794                         unformat_input_t * input, vlib_cli_command_t * cmd)
2795 {
2796   ip4_fib_t *fib;
2797   u32 table_id = 0;
2798   f64 count = 1;
2799   u32 n;
2800   int i;
2801   ip4_address_t ip4_base_address;
2802   u64 errors = 0;
2803
2804   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2805     {
2806       if (unformat (input, "table %d", &table_id))
2807         {
2808           /* Make sure the entry exists. */
2809           fib = ip4_fib_get (table_id);
2810           if ((fib) && (fib->index != table_id))
2811             return clib_error_return (0, "<fib-index> %d does not exist",
2812                                       table_id);
2813         }
2814       else if (unformat (input, "count %f", &count))
2815         ;
2816
2817       else if (unformat (input, "%U",
2818                          unformat_ip4_address, &ip4_base_address))
2819         ;
2820       else
2821         return clib_error_return (0, "unknown input `%U'",
2822                                   format_unformat_error, input);
2823     }
2824
2825   n = count;
2826
2827   for (i = 0; i < n; i++)
2828     {
2829       if (!ip4_lookup_validate (&ip4_base_address, table_id))
2830         errors++;
2831
2832       ip4_base_address.as_u32 =
2833         clib_host_to_net_u32 (1 +
2834                               clib_net_to_host_u32 (ip4_base_address.as_u32));
2835     }
2836
2837   if (errors)
2838     vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
2839   else
2840     vlib_cli_output (vm, "No errors in %d lookups\n", n);
2841
2842   return 0;
2843 }
2844
2845 /*?
2846  * Perform a lookup of an IPv4 Address (or range of addresses) in the
2847  * given FIB table to determine if there is a conflict with the
2848  * adjacency table. The fib-id can be determined by using the
2849  * '<em>show ip fib</em>' command. If fib-id is not entered, default value
2850  * of 0 is used.
2851  *
2852  * @todo This command uses fib-id, other commands use table-id (not
2853  * just a name, they are different indexes). Would like to change this
2854  * to table-id for consistency.
2855  *
2856  * @cliexpar
2857  * Example of how to run the test lookup command:
2858  * @cliexstart{test lookup 172.16.1.1 table 1 count 2}
2859  * No errors in 2 lookups
2860  * @cliexend
2861 ?*/
2862 /* *INDENT-OFF* */
2863 VLIB_CLI_COMMAND (lookup_test_command, static) =
2864 {
2865   .path = "test lookup",
2866   .short_help = "test lookup <ipv4-addr> [table <fib-id>] [count <nn>]",
2867   .function = test_lookup_command_fn,
2868 };
2869 /* *INDENT-ON* */
2870
2871 int
2872 vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
2873 {
2874   u32 fib_index;
2875
2876   fib_index = fib_table_find (FIB_PROTOCOL_IP4, table_id);
2877
2878   if (~0 == fib_index)
2879     return VNET_API_ERROR_NO_SUCH_FIB;
2880
2881   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP4,
2882                                   flow_hash_config);
2883
2884   return 0;
2885 }
2886
2887 static clib_error_t *
2888 set_ip_flow_hash_command_fn (vlib_main_t * vm,
2889                              unformat_input_t * input,
2890                              vlib_cli_command_t * cmd)
2891 {
2892   int matched = 0;
2893   u32 table_id = 0;
2894   u32 flow_hash_config = 0;
2895   int rv;
2896
2897   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2898     {
2899       if (unformat (input, "table %d", &table_id))
2900         matched = 1;
2901 #define _(a,v) \
2902     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2903       foreach_flow_hash_bit
2904 #undef _
2905         else
2906         break;
2907     }
2908
2909   if (matched == 0)
2910     return clib_error_return (0, "unknown input `%U'",
2911                               format_unformat_error, input);
2912
2913   rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
2914   switch (rv)
2915     {
2916     case 0:
2917       break;
2918
2919     case VNET_API_ERROR_NO_SUCH_FIB:
2920       return clib_error_return (0, "no such FIB table %d", table_id);
2921
2922     default:
2923       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2924       break;
2925     }
2926
2927   return 0;
2928 }
2929
2930 /*?
2931  * Configure the set of IPv4 fields used by the flow hash.
2932  *
2933  * @cliexpar
2934  * Example of how to set the flow hash on a given table:
2935  * @cliexcmd{set ip flow-hash table 7 dst sport dport proto}
2936  * Example of display the configured flow hash:
2937  * @cliexstart{show ip fib}
2938  * ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2939  * 0.0.0.0/0
2940  *   unicast-ip4-chain
2941  *   [@0]: dpo-load-balance: [index:0 buckets:1 uRPF:0 to:[0:0]]
2942  *     [0] [@0]: dpo-drop ip6
2943  * 0.0.0.0/32
2944  *   unicast-ip4-chain
2945  *   [@0]: dpo-load-balance: [index:1 buckets:1 uRPF:1 to:[0:0]]
2946  *     [0] [@0]: dpo-drop ip6
2947  * 224.0.0.0/8
2948  *   unicast-ip4-chain
2949  *   [@0]: dpo-load-balance: [index:3 buckets:1 uRPF:3 to:[0:0]]
2950  *     [0] [@0]: dpo-drop ip6
2951  * 6.0.1.2/32
2952  *   unicast-ip4-chain
2953  *   [@0]: dpo-load-balance: [index:30 buckets:1 uRPF:29 to:[0:0]]
2954  *     [0] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
2955  * 7.0.0.1/32
2956  *   unicast-ip4-chain
2957  *   [@0]: dpo-load-balance: [index:31 buckets:4 uRPF:30 to:[0:0]]
2958  *     [0] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
2959  *     [1] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
2960  *     [2] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
2961  *     [3] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
2962  * 240.0.0.0/8
2963  *   unicast-ip4-chain
2964  *   [@0]: dpo-load-balance: [index:2 buckets:1 uRPF:2 to:[0:0]]
2965  *     [0] [@0]: dpo-drop ip6
2966  * 255.255.255.255/32
2967  *   unicast-ip4-chain
2968  *   [@0]: dpo-load-balance: [index:4 buckets:1 uRPF:4 to:[0:0]]
2969  *     [0] [@0]: dpo-drop ip6
2970  * ipv4-VRF:7, fib_index 1, flow hash: dst sport dport proto
2971  * 0.0.0.0/0
2972  *   unicast-ip4-chain
2973  *   [@0]: dpo-load-balance: [index:12 buckets:1 uRPF:11 to:[0:0]]
2974  *     [0] [@0]: dpo-drop ip6
2975  * 0.0.0.0/32
2976  *   unicast-ip4-chain
2977  *   [@0]: dpo-load-balance: [index:13 buckets:1 uRPF:12 to:[0:0]]
2978  *     [0] [@0]: dpo-drop ip6
2979  * 172.16.1.0/24
2980  *   unicast-ip4-chain
2981  *   [@0]: dpo-load-balance: [index:17 buckets:1 uRPF:16 to:[0:0]]
2982  *     [0] [@4]: ipv4-glean: af_packet0
2983  * 172.16.1.1/32
2984  *   unicast-ip4-chain
2985  *   [@0]: dpo-load-balance: [index:18 buckets:1 uRPF:17 to:[1:84]]
2986  *     [0] [@2]: dpo-receive: 172.16.1.1 on af_packet0
2987  * 172.16.1.2/32
2988  *   unicast-ip4-chain
2989  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
2990  *     [0] [@5]: ipv4 via 172.16.1.2 af_packet0: IP4: 02:fe:9e:70:7a:2b -> 26:a5:f6:9c:3a:36
2991  * 172.16.2.0/24
2992  *   unicast-ip4-chain
2993  *   [@0]: dpo-load-balance: [index:19 buckets:1 uRPF:18 to:[0:0]]
2994  *     [0] [@4]: ipv4-glean: af_packet1
2995  * 172.16.2.1/32
2996  *   unicast-ip4-chain
2997  *   [@0]: dpo-load-balance: [index:20 buckets:1 uRPF:19 to:[0:0]]
2998  *     [0] [@2]: dpo-receive: 172.16.2.1 on af_packet1
2999  * 224.0.0.0/8
3000  *   unicast-ip4-chain
3001  *   [@0]: dpo-load-balance: [index:15 buckets:1 uRPF:14 to:[0:0]]
3002  *     [0] [@0]: dpo-drop ip6
3003  * 240.0.0.0/8
3004  *   unicast-ip4-chain
3005  *   [@0]: dpo-load-balance: [index:14 buckets:1 uRPF:13 to:[0:0]]
3006  *     [0] [@0]: dpo-drop ip6
3007  * 255.255.255.255/32
3008  *   unicast-ip4-chain
3009  *   [@0]: dpo-load-balance: [index:16 buckets:1 uRPF:15 to:[0:0]]
3010  *     [0] [@0]: dpo-drop ip6
3011  * @cliexend
3012 ?*/
3013 /* *INDENT-OFF* */
3014 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) =
3015 {
3016   .path = "set ip flow-hash",
3017   .short_help =
3018   "set ip flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
3019   .function = set_ip_flow_hash_command_fn,
3020 };
3021 /* *INDENT-ON* */
3022
3023 int
3024 vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3025                              u32 table_index)
3026 {
3027   vnet_main_t *vnm = vnet_get_main ();
3028   vnet_interface_main_t *im = &vnm->interface_main;
3029   ip4_main_t *ipm = &ip4_main;
3030   ip_lookup_main_t *lm = &ipm->lookup_main;
3031   vnet_classify_main_t *cm = &vnet_classify_main;
3032   ip4_address_t *if_addr;
3033
3034   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3035     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3036
3037   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3038     return VNET_API_ERROR_NO_SUCH_ENTRY;
3039
3040   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3041   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
3042
3043   if_addr = ip4_interface_first_address (ipm, sw_if_index, NULL);
3044
3045   if (NULL != if_addr)
3046     {
3047       fib_prefix_t pfx = {
3048         .fp_len = 32,
3049         .fp_proto = FIB_PROTOCOL_IP4,
3050         .fp_addr.ip4 = *if_addr,
3051       };
3052       u32 fib_index;
3053
3054       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3055                                                        sw_if_index);
3056
3057
3058       if (table_index != (u32) ~ 0)
3059         {
3060           dpo_id_t dpo = DPO_INVALID;
3061
3062           dpo_set (&dpo,
3063                    DPO_CLASSIFY,
3064                    DPO_PROTO_IP4,
3065                    classify_dpo_create (DPO_PROTO_IP4, table_index));
3066
3067           fib_table_entry_special_dpo_add (fib_index,
3068                                            &pfx,
3069                                            FIB_SOURCE_CLASSIFY,
3070                                            FIB_ENTRY_FLAG_NONE, &dpo);
3071           dpo_reset (&dpo);
3072         }
3073       else
3074         {
3075           fib_table_entry_special_remove (fib_index,
3076                                           &pfx, FIB_SOURCE_CLASSIFY);
3077         }
3078     }
3079
3080   return 0;
3081 }
3082
3083 static clib_error_t *
3084 set_ip_classify_command_fn (vlib_main_t * vm,
3085                             unformat_input_t * input,
3086                             vlib_cli_command_t * cmd)
3087 {
3088   u32 table_index = ~0;
3089   int table_index_set = 0;
3090   u32 sw_if_index = ~0;
3091   int rv;
3092
3093   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3094     {
3095       if (unformat (input, "table-index %d", &table_index))
3096         table_index_set = 1;
3097       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3098                          vnet_get_main (), &sw_if_index))
3099         ;
3100       else
3101         break;
3102     }
3103
3104   if (table_index_set == 0)
3105     return clib_error_return (0, "classify table-index must be specified");
3106
3107   if (sw_if_index == ~0)
3108     return clib_error_return (0, "interface / subif must be specified");
3109
3110   rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
3111
3112   switch (rv)
3113     {
3114     case 0:
3115       break;
3116
3117     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3118       return clib_error_return (0, "No such interface");
3119
3120     case VNET_API_ERROR_NO_SUCH_ENTRY:
3121       return clib_error_return (0, "No such classifier table");
3122     }
3123   return 0;
3124 }
3125
3126 /*?
3127  * Assign a classification table to an interface. The classification
3128  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3129  * commands. Once the table is create, use this command to filter packets
3130  * on an interface.
3131  *
3132  * @cliexpar
3133  * Example of how to assign a classification table to an interface:
3134  * @cliexcmd{set ip classify intfc GigabitEthernet2/0/0 table-index 1}
3135 ?*/
3136 /* *INDENT-OFF* */
3137 VLIB_CLI_COMMAND (set_ip_classify_command, static) =
3138 {
3139     .path = "set ip classify",
3140     .short_help =
3141     "set ip classify intfc <interface> table-index <classify-idx>",
3142     .function = set_ip_classify_command_fn,
3143 };
3144 /* *INDENT-ON* */
3145
3146 /*
3147  * fd.io coding-style-patch-verification: ON
3148  *
3149  * Local Variables:
3150  * eval: (c-set-style "gnu")
3151  * End:
3152  */