src/vnet/ip/ip4_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip4_forward.c: IP v4 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  43 #include <vnet/ethernet/arp_packet.h>   /* for ethernet_arp_header_t */
  44 #include <vnet/ppp/ppp.h>
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vnet/api_errno.h>     /* for API error numbers */
  47 #include <vnet/fib/fib_table.h> /* for FIB table and entry creation */
  48 #include <vnet/fib/fib_entry.h> /* for FIB table and entry creation */
  49 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  50 #include <vnet/fib/ip4_fib.h>
  51 #include <vnet/dpo/load_balance.h>
  52 #include <vnet/dpo/classify_dpo.h>
  53 #include <vnet/mfib/mfib_table.h>       /* for mFIB table and entry creation */
  54
  55 /**
  56  * @file
  57  * @brief IPv4 Forwarding.
  58  *
  59  * This file contains the source code for IPv4 forwarding.
  60  */
  61
  62 void
  63 ip4_forward_next_trace (vlib_main_t * vm,
  64                         vlib_node_runtime_t * node,
  65                         vlib_frame_t * frame,
  66                         vlib_rx_or_tx_t which_adj_index);
  67
  68 always_inline uword
  69 ip4_lookup_inline (vlib_main_t * vm,
  70                    vlib_node_runtime_t * node,
  71                    vlib_frame_t * frame,
  72                    int lookup_for_responses_to_locally_received_packets)
  73 {
  74   ip4_main_t *im = &ip4_main;
  75   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters;
  76   u32 n_left_from, n_left_to_next, *from, *to_next;
  77   ip_lookup_next_t next;
  78   u32 cpu_index = os_get_cpu_number ();
  79
  80   from = vlib_frame_vector_args (frame);
  81   n_left_from = frame->n_vectors;
  82   next = node->cached_next_index;
  83
  84   while (n_left_from > 0)
  85     {
  86       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
  87
  88       while (n_left_from >= 8 && n_left_to_next >= 4)
  89         {
  90           vlib_buffer_t *p0, *p1, *p2, *p3;
  91           ip4_header_t *ip0, *ip1, *ip2, *ip3;
  92           __attribute__ ((unused)) tcp_header_t *tcp0, *tcp1, *tcp2, *tcp3;
  93           ip_lookup_next_t next0, next1, next2, next3;
  94           const load_balance_t *lb0, *lb1, *lb2, *lb3;
  95           ip4_fib_mtrie_t *mtrie0, *mtrie1, *mtrie2, *mtrie3;
  96           ip4_fib_mtrie_leaf_t leaf0, leaf1, leaf2, leaf3;
  97           ip4_address_t *dst_addr0, *dst_addr1, *dst_addr2, *dst_addr3;
  98           u32 pi0, fib_index0, lb_index0;
  99           u32 pi1, fib_index1, lb_index1;
 100           u32 pi2, fib_index2, lb_index2;
 101           u32 pi3, fib_index3, lb_index3;
 102           flow_hash_config_t flow_hash_config0, flow_hash_config1;
 103           flow_hash_config_t flow_hash_config2, flow_hash_config3;
 104           u32 hash_c0, hash_c1, hash_c2, hash_c3;
 105           const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3;
 106
 107           /* Prefetch next iteration. */
 108           {
 109             vlib_buffer_t *p4, *p5, *p6, *p7;
 110
 111             p4 = vlib_get_buffer (vm, from[4]);
 112             p5 = vlib_get_buffer (vm, from[5]);
 113             p6 = vlib_get_buffer (vm, from[6]);
 114             p7 = vlib_get_buffer (vm, from[7]);
 115
 116             vlib_prefetch_buffer_header (p4, LOAD);
 117             vlib_prefetch_buffer_header (p5, LOAD);
 118             vlib_prefetch_buffer_header (p6, LOAD);
 119             vlib_prefetch_buffer_header (p7, LOAD);
 120
 121             CLIB_PREFETCH (p4->data, sizeof (ip0[0]), LOAD);
 122             CLIB_PREFETCH (p5->data, sizeof (ip0[0]), LOAD);
 123             CLIB_PREFETCH (p6->data, sizeof (ip0[0]), LOAD);
 124             CLIB_PREFETCH (p7->data, sizeof (ip0[0]), LOAD);
 125           }
 126
 127           pi0 = to_next[0] = from[0];
 128           pi1 = to_next[1] = from[1];
 129           pi2 = to_next[2] = from[2];
 130           pi3 = to_next[3] = from[3];
 131
 132           from += 4;
 133           to_next += 4;
 134           n_left_to_next -= 4;
 135           n_left_from -= 4;
 136
 137           p0 = vlib_get_buffer (vm, pi0);
 138           p1 = vlib_get_buffer (vm, pi1);
 139           p2 = vlib_get_buffer (vm, pi2);
 140           p3 = vlib_get_buffer (vm, pi3);
 141
 142           ip0 = vlib_buffer_get_current (p0);
 143           ip1 = vlib_buffer_get_current (p1);
 144           ip2 = vlib_buffer_get_current (p2);
 145           ip3 = vlib_buffer_get_current (p3);
 146
 147           dst_addr0 = &ip0->dst_address;
 148           dst_addr1 = &ip1->dst_address;
 149           dst_addr2 = &ip2->dst_address;
 150           dst_addr3 = &ip3->dst_address;
 151
 152           fib_index0 =
 153             vec_elt (im->fib_index_by_sw_if_index,
 154                      vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 155           fib_index1 =
 156             vec_elt (im->fib_index_by_sw_if_index,
 157                      vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 158           fib_index2 =
 159             vec_elt (im->fib_index_by_sw_if_index,
 160                      vnet_buffer (p2)->sw_if_index[VLIB_RX]);
 161           fib_index3 =
 162             vec_elt (im->fib_index_by_sw_if_index,
 163                      vnet_buffer (p3)->sw_if_index[VLIB_RX]);
 164           fib_index0 =
 165             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
 166              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 167           fib_index1 =
 168             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
 169              (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
 170           fib_index2 =
 171             (vnet_buffer (p2)->sw_if_index[VLIB_TX] ==
 172              (u32) ~ 0) ? fib_index2 : vnet_buffer (p2)->sw_if_index[VLIB_TX];
 173           fib_index3 =
 174             (vnet_buffer (p3)->sw_if_index[VLIB_TX] ==
 175              (u32) ~ 0) ? fib_index3 : vnet_buffer (p3)->sw_if_index[VLIB_TX];
 176
 177
 178           if (!lookup_for_responses_to_locally_received_packets)
 179             {
 180               mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 181               mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
 182               mtrie2 = &ip4_fib_get (fib_index2)->mtrie;
 183               mtrie3 = &ip4_fib_get (fib_index3)->mtrie;
 184
 185
 186               leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
 187               leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, dst_addr1);
 188               leaf2 = ip4_fib_mtrie_lookup_step_one (mtrie2, dst_addr2);
 189               leaf3 = ip4_fib_mtrie_lookup_step_one (mtrie3, dst_addr3);
 190             }
 191
 192           tcp0 = (void *) (ip0 + 1);
 193           tcp1 = (void *) (ip1 + 1);
 194           tcp2 = (void *) (ip2 + 1);
 195           tcp3 = (void *) (ip3 + 1);
 196
 197           if (!lookup_for_responses_to_locally_received_packets)
 198             {
 199               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
 200               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 1);
 201               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 1);
 202               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 1);
 203             }
 204
 205           if (!lookup_for_responses_to_locally_received_packets)
 206             {
 207               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 208               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
 209               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 2);
 210               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 2);
 211             }
 212
 213           if (!lookup_for_responses_to_locally_received_packets)
 214             {
 215               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 216               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
 217               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 3);
 218               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 3);
 219             }
 220
 221           if (lookup_for_responses_to_locally_received_packets)
 222             {
 223               lb_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 224               lb_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
 225               lb_index2 = vnet_buffer (p2)->ip.adj_index[VLIB_RX];
 226               lb_index3 = vnet_buffer (p3)->ip.adj_index[VLIB_RX];
 227             }
 228           else
 229             {
 230               lb_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 231               lb_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 232               lb_index2 = ip4_fib_mtrie_leaf_get_adj_index (leaf2);
 233               lb_index3 = ip4_fib_mtrie_leaf_get_adj_index (leaf3);
 234             }
 235
 236           ASSERT (lb_index0 && lb_index1 && lb_index2 && lb_index3);
 237           lb0 = load_balance_get (lb_index0);
 238           lb1 = load_balance_get (lb_index1);
 239           lb2 = load_balance_get (lb_index2);
 240           lb3 = load_balance_get (lb_index3);
 241
 242           /* Use flow hash to compute multipath adjacency. */
 243           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 244           hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
 245           hash_c2 = vnet_buffer (p2)->ip.flow_hash = 0;
 246           hash_c3 = vnet_buffer (p3)->ip.flow_hash = 0;
 247           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 248             {
 249               flow_hash_config0 = lb0->lb_hash_config;
 250               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 251                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 252             }
 253           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 254             {
 255               flow_hash_config1 = lb1->lb_hash_config;
 256               hash_c1 = vnet_buffer (p1)->ip.flow_hash =
 257                 ip4_compute_flow_hash (ip1, flow_hash_config1);
 258             }
 259           if (PREDICT_FALSE (lb2->lb_n_buckets > 1))
 260             {
 261               flow_hash_config2 = lb2->lb_hash_config;
 262               hash_c2 = vnet_buffer (p2)->ip.flow_hash =
 263                 ip4_compute_flow_hash (ip2, flow_hash_config2);
 264             }
 265           if (PREDICT_FALSE (lb3->lb_n_buckets > 1))
 266             {
 267               flow_hash_config3 = lb3->lb_hash_config;
 268               hash_c3 = vnet_buffer (p3)->ip.flow_hash =
 269                 ip4_compute_flow_hash (ip3, flow_hash_config3);
 270             }
 271
 272           ASSERT (lb0->lb_n_buckets > 0);
 273           ASSERT (is_pow2 (lb0->lb_n_buckets));
 274           ASSERT (lb1->lb_n_buckets > 0);
 275           ASSERT (is_pow2 (lb1->lb_n_buckets));
 276           ASSERT (lb2->lb_n_buckets > 0);
 277           ASSERT (is_pow2 (lb2->lb_n_buckets));
 278           ASSERT (lb3->lb_n_buckets > 0);
 279           ASSERT (is_pow2 (lb3->lb_n_buckets));
 280
 281           dpo0 = load_balance_get_bucket_i (lb0,
 282                                             (hash_c0 &
 283                                              (lb0->lb_n_buckets_minus_1)));
 284           dpo1 = load_balance_get_bucket_i (lb1,
 285                                             (hash_c1 &
 286                                              (lb1->lb_n_buckets_minus_1)));
 287           dpo2 = load_balance_get_bucket_i (lb2,
 288                                             (hash_c2 &
 289                                              (lb2->lb_n_buckets_minus_1)));
 290           dpo3 = load_balance_get_bucket_i (lb3,
 291                                             (hash_c3 &
 292                                              (lb3->lb_n_buckets_minus_1)));
 293
 294           next0 = dpo0->dpoi_next_node;
 295           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 296           next1 = dpo1->dpoi_next_node;
 297           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 298           next2 = dpo2->dpoi_next_node;
 299           vnet_buffer (p2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
 300           next3 = dpo3->dpoi_next_node;
 301           vnet_buffer (p3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
 302
 303           vlib_increment_combined_counter
 304             (cm, cpu_index, lb_index0, 1,
 305              vlib_buffer_length_in_chain (vm, p0)
 306              + sizeof (ethernet_header_t));
 307           vlib_increment_combined_counter
 308             (cm, cpu_index, lb_index1, 1,
 309              vlib_buffer_length_in_chain (vm, p1)
 310              + sizeof (ethernet_header_t));
 311           vlib_increment_combined_counter
 312             (cm, cpu_index, lb_index2, 1,
 313              vlib_buffer_length_in_chain (vm, p2)
 314              + sizeof (ethernet_header_t));
 315           vlib_increment_combined_counter
 316             (cm, cpu_index, lb_index3, 1,
 317              vlib_buffer_length_in_chain (vm, p3)
 318              + sizeof (ethernet_header_t));
 319
 320           vlib_validate_buffer_enqueue_x4 (vm, node, next,
 321                                            to_next, n_left_to_next,
 322                                            pi0, pi1, pi2, pi3,
 323                                            next0, next1, next2, next3);
 324         }
 325
 326       while (n_left_from > 0 && n_left_to_next > 0)
 327         {
 328           vlib_buffer_t *p0;
 329           ip4_header_t *ip0;
 330           __attribute__ ((unused)) tcp_header_t *tcp0;
 331           ip_lookup_next_t next0;
 332           const load_balance_t *lb0;
 333           ip4_fib_mtrie_t *mtrie0;
 334           ip4_fib_mtrie_leaf_t leaf0;
 335           ip4_address_t *dst_addr0;
 336           u32 pi0, fib_index0, lbi0;
 337           flow_hash_config_t flow_hash_config0;
 338           const dpo_id_t *dpo0;
 339           u32 hash_c0;
 340
 341           pi0 = from[0];
 342           to_next[0] = pi0;
 343
 344           p0 = vlib_get_buffer (vm, pi0);
 345
 346           ip0 = vlib_buffer_get_current (p0);
 347
 348           dst_addr0 = &ip0->dst_address;
 349
 350           fib_index0 =
 351             vec_elt (im->fib_index_by_sw_if_index,
 352                      vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 353           fib_index0 =
 354             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
 355              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 356
 357           if (!lookup_for_responses_to_locally_received_packets)
 358             {
 359               mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 360
 361               leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0);
 362             }
 363
 364           tcp0 = (void *) (ip0 + 1);
 365
 366           if (!lookup_for_responses_to_locally_received_packets)
 367             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
 368
 369           if (!lookup_for_responses_to_locally_received_packets)
 370             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 371
 372           if (!lookup_for_responses_to_locally_received_packets)
 373             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 374
 375           if (lookup_for_responses_to_locally_received_packets)
 376             lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 377           else
 378             {
 379               /* Handle default route. */
 380               lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 381             }
 382
 383           ASSERT (lbi0);
 384           lb0 = load_balance_get (lbi0);
 385
 386           /* Use flow hash to compute multipath adjacency. */
 387           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 388           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 389             {
 390               flow_hash_config0 = lb0->lb_hash_config;
 391
 392               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 393                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 394             }
 395
 396           ASSERT (lb0->lb_n_buckets > 0);
 397           ASSERT (is_pow2 (lb0->lb_n_buckets));
 398
 399           dpo0 = load_balance_get_bucket_i (lb0,
 400                                             (hash_c0 &
 401                                              (lb0->lb_n_buckets_minus_1)));
 402
 403           next0 = dpo0->dpoi_next_node;
 404           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 405
 406           vlib_increment_combined_counter
 407             (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 408
 409           from += 1;
 410           to_next += 1;
 411           n_left_to_next -= 1;
 412           n_left_from -= 1;
 413
 414           if (PREDICT_FALSE (next0 != next))
 415             {
 416               n_left_to_next += 1;
 417               vlib_put_next_frame (vm, node, next, n_left_to_next);
 418               next = next0;
 419               vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 420               to_next[0] = pi0;
 421               to_next += 1;
 422               n_left_to_next -= 1;
 423             }
 424         }
 425
 426       vlib_put_next_frame (vm, node, next, n_left_to_next);
 427     }
 428
 429   if (node->flags & VLIB_NODE_FLAG_TRACE)
 430     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 431
 432   return frame->n_vectors;
 433 }
 434
 435 /** @brief IPv4 lookup node.
 436     @node ip4-lookup
 437
 438     This is the main IPv4 lookup dispatch node.
 439
 440     @param vm vlib_main_t corresponding to the current thread
 441     @param node vlib_node_runtime_t
 442     @param frame vlib_frame_t whose contents should be dispatched
 443
 444     @par Graph mechanics: buffer metadata, next index usage
 445
 446     @em Uses:
 447     - <code>vnet_buffer(b)->sw_if_index[VLIB_RX]</code>
 448         - Indicates the @c sw_if_index value of the interface that the
 449           packet was received on.
 450     - <code>vnet_buffer(b)->sw_if_index[VLIB_TX]</code>
 451         - When the value is @c ~0 then the node performs a longest prefix
 452           match (LPM) for the packet destination address in the FIB attached
 453           to the receive interface.
 454         - Otherwise perform LPM for the packet destination address in the
 455           indicated FIB. In this case <code>[VLIB_TX]</code> is a FIB index
 456           value (0, 1, ...) and not a VRF id.
 457
 458     @em Sets:
 459     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
 460         - The lookup result adjacency index.
 461
 462     <em>Next Index:</em>
 463     - Dispatches the packet to the node index found in
 464       ip_adjacency_t @c adj->lookup_next_index
 465       (where @c adj is the lookup result adjacency).
 466 */
 467 static uword
 468 ip4_lookup (vlib_main_t * vm,
 469             vlib_node_runtime_t * node, vlib_frame_t * frame)
 470 {
 471   return ip4_lookup_inline (vm, node, frame,
 472                             /* lookup_for_responses_to_locally_received_packets */
 473                             0);
 474
 475 }
 476
 477 static u8 *format_ip4_lookup_trace (u8 * s, va_list * args);
 478
 479 VLIB_REGISTER_NODE (ip4_lookup_node) =
 480 {
 481 .function = ip4_lookup,.name = "ip4-lookup",.vector_size =
 482     sizeof (u32),.format_trace = format_ip4_lookup_trace,.n_next_nodes =
 483     IP_LOOKUP_N_NEXT,.next_nodes = IP4_LOOKUP_NEXT_NODES,};
 484
 485 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup);
 486
 487 always_inline uword
 488 ip4_load_balance (vlib_main_t * vm,
 489                   vlib_node_runtime_t * node, vlib_frame_t * frame)
 490 {
 491   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 492   u32 n_left_from, n_left_to_next, *from, *to_next;
 493   ip_lookup_next_t next;
 494   u32 cpu_index = os_get_cpu_number ();
 495
 496   from = vlib_frame_vector_args (frame);
 497   n_left_from = frame->n_vectors;
 498   next = node->cached_next_index;
 499
 500   if (node->flags & VLIB_NODE_FLAG_TRACE)
 501     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 502
 503   while (n_left_from > 0)
 504     {
 505       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 506
 507
 508       while (n_left_from >= 4 && n_left_to_next >= 2)
 509         {
 510           ip_lookup_next_t next0, next1;
 511           const load_balance_t *lb0, *lb1;
 512           vlib_buffer_t *p0, *p1;
 513           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 514           const ip4_header_t *ip0, *ip1;
 515           const dpo_id_t *dpo0, *dpo1;
 516
 517           /* Prefetch next iteration. */
 518           {
 519             vlib_buffer_t *p2, *p3;
 520
 521             p2 = vlib_get_buffer (vm, from[2]);
 522             p3 = vlib_get_buffer (vm, from[3]);
 523
 524             vlib_prefetch_buffer_header (p2, STORE);
 525             vlib_prefetch_buffer_header (p3, STORE);
 526
 527             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 528             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 529           }
 530
 531           pi0 = to_next[0] = from[0];
 532           pi1 = to_next[1] = from[1];
 533
 534           from += 2;
 535           n_left_from -= 2;
 536           to_next += 2;
 537           n_left_to_next -= 2;
 538
 539           p0 = vlib_get_buffer (vm, pi0);
 540           p1 = vlib_get_buffer (vm, pi1);
 541
 542           ip0 = vlib_buffer_get_current (p0);
 543           ip1 = vlib_buffer_get_current (p1);
 544           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 545           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 546
 547           lb0 = load_balance_get (lbi0);
 548           lb1 = load_balance_get (lbi1);
 549
 550           /*
 551            * this node is for via FIBs we can re-use the hash value from the
 552            * to node if present.
 553            * We don't want to use the same hash value at each level in the recursion
 554            * graph as that would lead to polarisation
 555            */
 556           hc0 = hc1 = 0;
 557
 558           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 559             {
 560               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 561                 {
 562                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 563                     vnet_buffer (p0)->ip.flow_hash >> 1;
 564                 }
 565               else
 566                 {
 567                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 568                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 569                 }
 570             }
 571           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 572             {
 573               if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash))
 574                 {
 575                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 576                     vnet_buffer (p1)->ip.flow_hash >> 1;
 577                 }
 578               else
 579                 {
 580                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 581                     ip4_compute_flow_hash (ip1, lb1->lb_hash_config);
 582                 }
 583             }
 584
 585           dpo0 =
 586             load_balance_get_bucket_i (lb0,
 587                                        hc0 & (lb0->lb_n_buckets_minus_1));
 588           dpo1 =
 589             load_balance_get_bucket_i (lb1,
 590                                        hc1 & (lb1->lb_n_buckets_minus_1));
 591
 592           next0 = dpo0->dpoi_next_node;
 593           next1 = dpo1->dpoi_next_node;
 594
 595           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 596           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 597
 598           vlib_increment_combined_counter
 599             (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 600           vlib_increment_combined_counter
 601             (cm, cpu_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 602
 603           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 604                                            to_next, n_left_to_next,
 605                                            pi0, pi1, next0, next1);
 606         }
 607
 608       while (n_left_from > 0 && n_left_to_next > 0)
 609         {
 610           ip_lookup_next_t next0;
 611           const load_balance_t *lb0;
 612           vlib_buffer_t *p0;
 613           u32 pi0, lbi0, hc0;
 614           const ip4_header_t *ip0;
 615           const dpo_id_t *dpo0;
 616
 617           pi0 = from[0];
 618           to_next[0] = pi0;
 619           from += 1;
 620           to_next += 1;
 621           n_left_to_next -= 1;
 622           n_left_from -= 1;
 623
 624           p0 = vlib_get_buffer (vm, pi0);
 625
 626           ip0 = vlib_buffer_get_current (p0);
 627           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 628
 629           lb0 = load_balance_get (lbi0);
 630
 631           hc0 = 0;
 632           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 633             {
 634               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 635                 {
 636                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 637                     vnet_buffer (p0)->ip.flow_hash >> 1;
 638                 }
 639               else
 640                 {
 641                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 642                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 643                 }
 644             }
 645
 646           dpo0 =
 647             load_balance_get_bucket_i (lb0,
 648                                        hc0 & (lb0->lb_n_buckets_minus_1));
 649
 650           next0 = dpo0->dpoi_next_node;
 651           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 652
 653           vlib_increment_combined_counter
 654             (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 655
 656           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 657                                            to_next, n_left_to_next,
 658                                            pi0, next0);
 659         }
 660
 661       vlib_put_next_frame (vm, node, next, n_left_to_next);
 662     }
 663
 664   return frame->n_vectors;
 665 }
 666
 667 VLIB_REGISTER_NODE (ip4_load_balance_node) =
 668 {
 669 .function = ip4_load_balance,.name = "ip4-load-balance",.vector_size =
 670     sizeof (u32),.sibling_of = "ip4-lookup",.format_trace =
 671     format_ip4_lookup_trace,};
 672
 673 VLIB_NODE_FUNCTION_MULTIARCH (ip4_load_balance_node, ip4_load_balance);
 674
 675 /* get first interface address */
 676 ip4_address_t *
 677 ip4_interface_first_address (ip4_main_t * im, u32 sw_if_index,
 678                              ip_interface_address_t ** result_ia)
 679 {
 680   ip_lookup_main_t *lm = &im->lookup_main;
 681   ip_interface_address_t *ia = 0;
 682   ip4_address_t *result = 0;
 683
 684   /* *INDENT-OFF* */
 685   foreach_ip_interface_address
 686     (lm, ia, sw_if_index,
 687      1 /* honor unnumbered */ ,
 688      ({
 689        ip4_address_t * a =
 690          ip_interface_address_get_address (lm, ia);
 691        result = a;
 692        break;
 693      }));
 694   /* *INDENT-OFF* */
 695   if (result_ia)
 696     *result_ia = result ? ia : 0;
 697   return result;
 698 }
 699
 700 static void
 701 ip4_add_interface_routes (u32 sw_if_index,
 702                           ip4_main_t * im, u32 fib_index,
 703                           ip_interface_address_t * a)
 704 {
 705   ip_lookup_main_t *lm = &im->lookup_main;
 706   ip4_address_t *address = ip_interface_address_get_address (lm, a);
 707   fib_prefix_t pfx = {
 708     .fp_len = a->address_length,
 709     .fp_proto = FIB_PROTOCOL_IP4,
 710     .fp_addr.ip4 = *address,
 711   };
 712
 713   a->neighbor_probe_adj_index = ~0;
 714
 715   if (pfx.fp_len <= 30)
 716     {
 717       /* a /30 or shorter - add a glean for the network address */
 718       fib_node_index_t fei;
 719
 720       fei = fib_table_entry_update_one_path (fib_index, &pfx,
 721                                              FIB_SOURCE_INTERFACE,
 722                                              (FIB_ENTRY_FLAG_CONNECTED |
 723                                               FIB_ENTRY_FLAG_ATTACHED),
 724                                              FIB_PROTOCOL_IP4,
 725                                              /* No next-hop address */
 726                                              NULL,
 727                                              sw_if_index,
 728                                              // invalid FIB index
 729                                              ~0,
 730                                              1,
 731                                              // no out-label stack
 732                                              NULL,
 733                                              FIB_ROUTE_PATH_FLAG_NONE);
 734       a->neighbor_probe_adj_index = fib_entry_get_adj (fei);
 735
 736       /* Add the two broadcast addresses as drop */
 737       fib_prefix_t net_pfx = {
 738         .fp_len = 32,
 739         .fp_proto = FIB_PROTOCOL_IP4,
 740         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 741       };
 742       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 743         fib_table_entry_special_add(fib_index,
 744                                     &net_pfx,
 745                                     FIB_SOURCE_INTERFACE,
 746                                     (FIB_ENTRY_FLAG_DROP |
 747                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT),
 748                                     ADJ_INDEX_INVALID);
 749       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 750       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 751         fib_table_entry_special_add(fib_index,
 752                                     &net_pfx,
 753                                     FIB_SOURCE_INTERFACE,
 754                                     (FIB_ENTRY_FLAG_DROP |
 755                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT),
 756                                     ADJ_INDEX_INVALID);
 757     }
 758   else if (pfx.fp_len == 31)
 759     {
 760       u32 mask = clib_host_to_net_u32(1);
 761       fib_prefix_t net_pfx = pfx;
 762
 763       net_pfx.fp_len = 32;
 764       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 765
 766       /* a /31 - add the other end as an attached host */
 767       fib_table_entry_update_one_path (fib_index, &net_pfx,
 768                                        FIB_SOURCE_INTERFACE,
 769                                        (FIB_ENTRY_FLAG_ATTACHED),
 770                                        FIB_PROTOCOL_IP4,
 771                                        &net_pfx.fp_addr,
 772                                        sw_if_index,
 773                                        // invalid FIB index
 774                                        ~0,
 775                                        1,
 776                                        NULL,
 777                                        FIB_ROUTE_PATH_FLAG_NONE);
 778     }
 779   pfx.fp_len = 32;
 780
 781   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 782     {
 783       u32 classify_table_index =
 784         lm->classify_table_index_by_sw_if_index[sw_if_index];
 785       if (classify_table_index != (u32) ~ 0)
 786         {
 787           dpo_id_t dpo = DPO_INVALID;
 788
 789           dpo_set (&dpo,
 790                    DPO_CLASSIFY,
 791                    DPO_PROTO_IP4,
 792                    classify_dpo_create (DPO_PROTO_IP4, classify_table_index));
 793
 794           fib_table_entry_special_dpo_add (fib_index,
 795                                            &pfx,
 796                                            FIB_SOURCE_CLASSIFY,
 797                                            FIB_ENTRY_FLAG_NONE, &dpo);
 798           dpo_reset (&dpo);
 799         }
 800     }
 801
 802   fib_table_entry_update_one_path (fib_index, &pfx,
 803                                    FIB_SOURCE_INTERFACE,
 804                                    (FIB_ENTRY_FLAG_CONNECTED |
 805                                     FIB_ENTRY_FLAG_LOCAL),
 806                                    FIB_PROTOCOL_IP4,
 807                                    &pfx.fp_addr,
 808                                    sw_if_index,
 809                                    // invalid FIB index
 810                                    ~0,
 811                                    1, NULL,
 812                                    FIB_ROUTE_PATH_FLAG_NONE);
 813 }
 814
 815 static void
 816 ip4_del_interface_routes (ip4_main_t * im,
 817                           u32 fib_index,
 818                           ip4_address_t * address, u32 address_length)
 819 {
 820   fib_prefix_t pfx = {
 821     .fp_len = address_length,
 822     .fp_proto = FIB_PROTOCOL_IP4,
 823     .fp_addr.ip4 = *address,
 824   };
 825
 826   if (pfx.fp_len <= 30)
 827     {
 828       fib_prefix_t net_pfx = {
 829         .fp_len = 32,
 830         .fp_proto = FIB_PROTOCOL_IP4,
 831         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 832       };
 833       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 834         fib_table_entry_special_remove(fib_index,
 835                                        &net_pfx,
 836                                        FIB_SOURCE_INTERFACE);
 837       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 838       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 839         fib_table_entry_special_remove(fib_index,
 840                                        &net_pfx,
 841                                        FIB_SOURCE_INTERFACE);
 842       fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 843     }
 844     else if (pfx.fp_len == 31)
 845     {
 846       u32 mask = clib_host_to_net_u32(1);
 847       fib_prefix_t net_pfx = pfx;
 848
 849       net_pfx.fp_len = 32;
 850       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 851
 852       fib_table_entry_delete (fib_index, &net_pfx, FIB_SOURCE_INTERFACE);
 853     }
 854
 855   pfx.fp_len = 32;
 856   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 857 }
 858
 859 void
 860 ip4_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 861 {
 862   ip4_main_t *im = &ip4_main;
 863
 864   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 865
 866   /*
 867    * enable/disable only on the 1<->0 transition
 868    */
 869   if (is_enable)
 870     {
 871       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 872         return;
 873     }
 874   else
 875     {
 876       ASSERT (im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 877       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 878         return;
 879     }
 880   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
 881                                !is_enable, 0, 0);
 882
 883
 884   vnet_feature_enable_disable ("ip4-multicast", "ip4-drop",
 885                                sw_if_index, !is_enable, 0, 0);
 886 }
 887
 888 static clib_error_t *
 889 ip4_add_del_interface_address_internal (vlib_main_t * vm,
 890                                         u32 sw_if_index,
 891                                         ip4_address_t * address,
 892                                         u32 address_length, u32 is_del)
 893 {
 894   vnet_main_t *vnm = vnet_get_main ();
 895   ip4_main_t *im = &ip4_main;
 896   ip_lookup_main_t *lm = &im->lookup_main;
 897   clib_error_t *error = 0;
 898   u32 if_address_index, elts_before;
 899   ip4_address_fib_t ip4_af, *addr_fib = 0;
 900
 901   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 902   ip4_addr_fib_init (&ip4_af, address,
 903                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 904   vec_add1 (addr_fib, ip4_af);
 905
 906   /* FIXME-LATER
 907    * there is no support for adj-fib handling in the presence of overlapping
 908    * subnets on interfaces. Easy fix - disallow overlapping subnets, like
 909    * most routers do.
 910    */
 911   /* *INDENT-OFF* */
 912   if (!is_del)
 913     {
 914       /* When adding an address check that it does not conflict
 915          with an existing address. */
 916       ip_interface_address_t *ia;
 917       foreach_ip_interface_address
 918         (&im->lookup_main, ia, sw_if_index,
 919          0 /* honor unnumbered */ ,
 920          ({
 921            ip4_address_t * x =
 922              ip_interface_address_get_address
 923              (&im->lookup_main, ia);
 924            if (ip4_destination_matches_route
 925                (im, address, x, ia->address_length) ||
 926                ip4_destination_matches_route (im,
 927                                               x,
 928                                               address,
 929                                               address_length))
 930              return
 931                clib_error_create
 932                ("failed to add %U which conflicts with %U for interface %U",
 933                 format_ip4_address_and_length, address,
 934                 address_length,
 935                 format_ip4_address_and_length, x,
 936                 ia->address_length,
 937                 format_vnet_sw_if_index_name, vnm,
 938                 sw_if_index);
 939          }));
 940     }
 941   /* *INDENT-ON* */
 942
 943   elts_before = pool_elts (lm->if_address_pool);
 944
 945   error = ip_interface_address_add_del
 946     (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 947   if (error)
 948     goto done;
 949
 950   ip4_sw_interface_enable_disable (sw_if_index, !is_del);
 951
 952   if (is_del)
 953     ip4_del_interface_routes (im, ip4_af.fib_index, address, address_length);
 954   else
 955     ip4_add_interface_routes (sw_if_index,
 956                               im, ip4_af.fib_index,
 957                               pool_elt_at_index
 958                               (lm->if_address_pool, if_address_index));
 959
 960   /* If pool did not grow/shrink: add duplicate address. */
 961   if (elts_before != pool_elts (lm->if_address_pool))
 962     {
 963       ip4_add_del_interface_address_callback_t *cb;
 964       vec_foreach (cb, im->add_del_interface_address_callbacks)
 965         cb->function (im, cb->function_opaque, sw_if_index,
 966                       address, address_length, if_address_index, is_del);
 967     }
 968
 969 done:
 970   vec_free (addr_fib);
 971   return error;
 972 }
 973
 974 clib_error_t *
 975 ip4_add_del_interface_address (vlib_main_t * vm,
 976                                u32 sw_if_index,
 977                                ip4_address_t * address,
 978                                u32 address_length, u32 is_del)
 979 {
 980   return ip4_add_del_interface_address_internal
 981     (vm, sw_if_index, address, address_length, is_del);
 982 }
 983
 984 /* Built-in ip4 unicast rx feature path definition */
 985 /* *INDENT-OFF* */
 986 VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
 987 {
 988   .arc_name = "ip4-unicast",
 989   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 990   .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
 991 };
 992
 993 VNET_FEATURE_INIT (ip4_flow_classify, static) =
 994 {
 995   .arc_name = "ip4-unicast",
 996   .node_name = "ip4-flow-classify",
 997   .runs_before = VNET_FEATURES ("ip4-inacl"),
 998 };
 999
1000 VNET_FEATURE_INIT (ip4_inacl, static) =
1001 {
1002   .arc_name = "ip4-unicast",
1003   .node_name = "ip4-inacl",
1004   .runs_before = VNET_FEATURES ("ip4-source-check-via-rx"),
1005 };
1006
1007 VNET_FEATURE_INIT (ip4_source_check_1, static) =
1008 {
1009   .arc_name = "ip4-unicast",
1010   .node_name = "ip4-source-check-via-rx",
1011   .runs_before = VNET_FEATURES ("ip4-source-check-via-any"),
1012 };
1013
1014 VNET_FEATURE_INIT (ip4_source_check_2, static) =
1015 {
1016   .arc_name = "ip4-unicast",
1017   .node_name = "ip4-source-check-via-any",
1018   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
1019 };
1020
1021 VNET_FEATURE_INIT (ip4_source_and_port_range_check_rx, static) =
1022 {
1023   .arc_name = "ip4-unicast",
1024   .node_name = "ip4-source-and-port-range-check-rx",
1025   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
1026 };
1027
1028 VNET_FEATURE_INIT (ip4_policer_classify, static) =
1029 {
1030   .arc_name = "ip4-unicast",
1031   .node_name = "ip4-policer-classify",
1032   .runs_before = VNET_FEATURES ("ipsec-input-ip4"),
1033 };
1034
1035 VNET_FEATURE_INIT (ip4_ipsec, static) =
1036 {
1037   .arc_name = "ip4-unicast",
1038   .node_name = "ipsec-input-ip4",
1039   .runs_before = VNET_FEATURES ("vpath-input-ip4"),
1040 };
1041
1042 VNET_FEATURE_INIT (ip4_vpath, static) =
1043 {
1044   .arc_name = "ip4-unicast",
1045   .node_name = "vpath-input-ip4",
1046   .runs_before = VNET_FEATURES ("ip4-vxlan-bypass"),
1047 };
1048
1049 VNET_FEATURE_INIT (ip4_vxlan_bypass, static) =
1050 {
1051   .arc_name = "ip4-unicast",
1052   .node_name = "ip4-vxlan-bypass",
1053   .runs_before = VNET_FEATURES ("ip4-lookup"),
1054 };
1055
1056 VNET_FEATURE_INIT (ip4_drop, static) =
1057 {
1058   .arc_name = "ip4-unicast",
1059   .node_name = "ip4-drop",
1060   .runs_before = VNET_FEATURES ("ip4-lookup"),
1061 };
1062
1063 VNET_FEATURE_INIT (ip4_lookup, static) =
1064 {
1065   .arc_name = "ip4-unicast",
1066   .node_name = "ip4-lookup",
1067   .runs_before = 0,     /* not before any other features */
1068 };
1069
1070 /* Built-in ip4 multicast rx feature path definition */
1071 VNET_FEATURE_ARC_INIT (ip4_multicast, static) =
1072 {
1073   .arc_name = "ip4-multicast",
1074   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
1075   .arc_index_ptr = &ip4_main.lookup_main.mcast_feature_arc_index,
1076 };
1077
1078 VNET_FEATURE_INIT (ip4_vpath_mc, static) =
1079 {
1080   .arc_name = "ip4-multicast",
1081   .node_name = "vpath-input-ip4",
1082   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
1083 };
1084
1085 VNET_FEATURE_INIT (ip4_mc_drop, static) =
1086 {
1087   .arc_name = "ip4-multicast",
1088   .node_name = "ip4-drop",
1089   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
1090 };
1091
1092 VNET_FEATURE_INIT (ip4_lookup_mc, static) =
1093 {
1094   .arc_name = "ip4-multicast",
1095   .node_name = "ip4-mfib-forward-lookup",
1096   .runs_before = 0,     /* last feature */
1097 };
1098
1099 /* Source and port-range check ip4 tx feature path definition */
1100 VNET_FEATURE_ARC_INIT (ip4_output, static) =
1101 {
1102   .arc_name = "ip4-output",
1103   .start_nodes = VNET_FEATURES ("ip4-rewrite", "ip4-midchain"),
1104   .arc_index_ptr = &ip4_main.lookup_main.output_feature_arc_index,
1105 };
1106
1107 VNET_FEATURE_INIT (ip4_source_and_port_range_check_tx, static) =
1108 {
1109   .arc_name = "ip4-output",
1110   .node_name = "ip4-source-and-port-range-check-tx",
1111   .runs_before = VNET_FEATURES ("ipsec-output-ip4"),
1112 };
1113
1114 VNET_FEATURE_INIT (ip4_ipsec_output, static) =
1115 {
1116   .arc_name = "ip4-output",
1117   .node_name = "ipsec-output-ip4",
1118   .runs_before = VNET_FEATURES ("interface-output"),
1119 };
1120
1121 /* Built-in ip4 tx feature path definition */
1122 VNET_FEATURE_INIT (ip4_interface_output, static) =
1123 {
1124   .arc_name = "ip4-output",
1125   .node_name = "interface-output",
1126   .runs_before = 0,     /* not before any other features */
1127 };
1128 /* *INDENT-ON* */
1129
1130 static clib_error_t *
1131 ip4_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
1132 {
1133   ip4_main_t *im = &ip4_main;
1134
1135   /* Fill in lookup tables with default table (0). */
1136   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1137   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
1138
1139   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
1140                                is_add, 0, 0);
1141
1142   vnet_feature_enable_disable ("ip4-multicast", "ip4-drop", sw_if_index,
1143                                is_add, 0, 0);
1144
1145   return /* no error */ 0;
1146 }
1147
1148 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
1149
1150 /* Global IP4 main. */
1151 ip4_main_t ip4_main;
1152
1153 clib_error_t *
1154 ip4_lookup_init (vlib_main_t * vm)
1155 {
1156   ip4_main_t *im = &ip4_main;
1157   clib_error_t *error;
1158   uword i;
1159
1160   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
1161     return error;
1162
1163   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
1164     {
1165       u32 m;
1166
1167       if (i < 32)
1168         m = pow2_mask (i) << (32 - i);
1169       else
1170         m = ~0;
1171       im->fib_masks[i] = clib_host_to_net_u32 (m);
1172     }
1173
1174   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
1175
1176   /* Create FIB with index 0 and table id of 0. */
1177   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0);
1178   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0);
1179
1180   {
1181     pg_node_t *pn;
1182     pn = pg_get_node (ip4_lookup_node.index);
1183     pn->unformat_edit = unformat_pg_ip4_header;
1184   }
1185
1186   {
1187     ethernet_arp_header_t h;
1188
1189     memset (&h, 0, sizeof (h));
1190
1191     /* Set target ethernet address to all zeros. */
1192     memset (h.ip4_over_ethernet[1].ethernet, 0,
1193             sizeof (h.ip4_over_ethernet[1].ethernet));
1194
1195 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
1196 #define _8(f,v) h.f = v;
1197     _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
1198     _16 (l3_type, ETHERNET_TYPE_IP4);
1199     _8 (n_l2_address_bytes, 6);
1200     _8 (n_l3_address_bytes, 4);
1201     _16 (opcode, ETHERNET_ARP_OPCODE_request);
1202 #undef _16
1203 #undef _8
1204
1205     vlib_packet_template_init (vm, &im->ip4_arp_request_packet_template,
1206                                /* data */ &h,
1207                                sizeof (h),
1208                                /* alloc chunk size */ 8,
1209                                "ip4 arp");
1210   }
1211
1212   return error;
1213 }
1214
1215 VLIB_INIT_FUNCTION (ip4_lookup_init);
1216
1217 typedef struct
1218 {
1219   /* Adjacency taken. */
1220   u32 dpo_index;
1221   u32 flow_hash;
1222   u32 fib_index;
1223
1224   /* Packet data, possibly *after* rewrite. */
1225   u8 packet_data[64 - 1 * sizeof (u32)];
1226 }
1227 ip4_forward_next_trace_t;
1228
1229 u8 *
1230 format_ip4_forward_next_trace (u8 * s, va_list * args)
1231 {
1232   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1233   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1234   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1235   uword indent = format_get_indent (s);
1236   s = format (s, "%U%U",
1237               format_white_space, indent,
1238               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1239   return s;
1240 }
1241
1242 static u8 *
1243 format_ip4_lookup_trace (u8 * s, va_list * args)
1244 {
1245   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1246   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1247   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1248   uword indent = format_get_indent (s);
1249
1250   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
1251               t->fib_index, t->dpo_index, t->flow_hash);
1252   s = format (s, "\n%U%U",
1253               format_white_space, indent,
1254               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1255   return s;
1256 }
1257
1258 static u8 *
1259 format_ip4_rewrite_trace (u8 * s, va_list * args)
1260 {
1261   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1262   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1263   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1264   uword indent = format_get_indent (s);
1265
1266   s = format (s, "tx_sw_if_index %d dpo-idx %d : %U flow hash: 0x%08x",
1267               t->fib_index, t->dpo_index, format_ip_adjacency,
1268               t->dpo_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
1269   s = format (s, "\n%U%U",
1270               format_white_space, indent,
1271               format_ip_adjacency_packet_data,
1272               t->dpo_index, t->packet_data, sizeof (t->packet_data));
1273   return s;
1274 }
1275
1276 /* Common trace function for all ip4-forward next nodes. */
1277 void
1278 ip4_forward_next_trace (vlib_main_t * vm,
1279                         vlib_node_runtime_t * node,
1280                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
1281 {
1282   u32 *from, n_left;
1283   ip4_main_t *im = &ip4_main;
1284
1285   n_left = frame->n_vectors;
1286   from = vlib_frame_vector_args (frame);
1287
1288   while (n_left >= 4)
1289     {
1290       u32 bi0, bi1;
1291       vlib_buffer_t *b0, *b1;
1292       ip4_forward_next_trace_t *t0, *t1;
1293
1294       /* Prefetch next iteration. */
1295       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1296       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1297
1298       bi0 = from[0];
1299       bi1 = from[1];
1300
1301       b0 = vlib_get_buffer (vm, bi0);
1302       b1 = vlib_get_buffer (vm, bi1);
1303
1304       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1305         {
1306           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1307           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1308           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1309           t0->fib_index =
1310             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1311              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1312             vec_elt (im->fib_index_by_sw_if_index,
1313                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1314
1315           clib_memcpy (t0->packet_data,
1316                        vlib_buffer_get_current (b0),
1317                        sizeof (t0->packet_data));
1318         }
1319       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1320         {
1321           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1322           t1->dpo_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1323           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1324           t1->fib_index =
1325             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
1326              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
1327             vec_elt (im->fib_index_by_sw_if_index,
1328                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
1329           clib_memcpy (t1->packet_data, vlib_buffer_get_current (b1),
1330                        sizeof (t1->packet_data));
1331         }
1332       from += 2;
1333       n_left -= 2;
1334     }
1335
1336   while (n_left >= 1)
1337     {
1338       u32 bi0;
1339       vlib_buffer_t *b0;
1340       ip4_forward_next_trace_t *t0;
1341
1342       bi0 = from[0];
1343
1344       b0 = vlib_get_buffer (vm, bi0);
1345
1346       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1347         {
1348           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1349           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1350           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1351           t0->fib_index =
1352             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1353              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1354             vec_elt (im->fib_index_by_sw_if_index,
1355                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1356           clib_memcpy (t0->packet_data, vlib_buffer_get_current (b0),
1357                        sizeof (t0->packet_data));
1358         }
1359       from += 1;
1360       n_left -= 1;
1361     }
1362 }
1363
1364 static uword
1365 ip4_drop_or_punt (vlib_main_t * vm,
1366                   vlib_node_runtime_t * node,
1367                   vlib_frame_t * frame, ip4_error_t error_code)
1368 {
1369   u32 *buffers = vlib_frame_vector_args (frame);
1370   uword n_packets = frame->n_vectors;
1371
1372   vlib_error_drop_buffers (vm, node, buffers,
1373                            /* stride */ 1,
1374                            n_packets,
1375                            /* next */ 0,
1376                            ip4_input_node.index, error_code);
1377
1378   if (node->flags & VLIB_NODE_FLAG_TRACE)
1379     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1380
1381   return n_packets;
1382 }
1383
1384 static uword
1385 ip4_drop (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1386 {
1387   return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_DROP);
1388 }
1389
1390 static uword
1391 ip4_punt (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1392 {
1393   return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_PUNT);
1394 }
1395
1396 /* *INDENT-OFF* */
1397 VLIB_REGISTER_NODE (ip4_drop_node, static) =
1398 {
1399   .function = ip4_drop,.
1400   name = "ip4-drop",
1401   .vector_size = sizeof (u32),
1402   .format_trace = format_ip4_forward_next_trace,
1403   .n_next_nodes = 1,
1404   .next_nodes = {
1405     [0] = "error-drop",
1406   },
1407 };
1408
1409 VLIB_NODE_FUNCTION_MULTIARCH (ip4_drop_node, ip4_drop);
1410
1411 VLIB_REGISTER_NODE (ip4_punt_node, static) =
1412 {
1413   .function = ip4_punt,
1414   .name = "ip4-punt",
1415   .vector_size = sizeof (u32),
1416   .format_trace = format_ip4_forward_next_trace,
1417   .n_next_nodes = 1,
1418   .next_nodes = {
1419     [0] = "error-punt",
1420   },
1421 };
1422
1423 VLIB_NODE_FUNCTION_MULTIARCH (ip4_punt_node, ip4_punt);
1424 /* *INDENT-ON */
1425
1426 /* Compute TCP/UDP/ICMP4 checksum in software. */
1427 u16
1428 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1429                               ip4_header_t * ip0)
1430 {
1431   ip_csum_t sum0;
1432   u32 ip_header_length, payload_length_host_byte_order;
1433   u32 n_this_buffer, n_bytes_left;
1434   u16 sum16;
1435   void *data_this_buffer;
1436
1437   /* Initialize checksum with ip header. */
1438   ip_header_length = ip4_header_bytes (ip0);
1439   payload_length_host_byte_order =
1440     clib_net_to_host_u16 (ip0->length) - ip_header_length;
1441   sum0 =
1442     clib_host_to_net_u32 (payload_length_host_byte_order +
1443                           (ip0->protocol << 16));
1444
1445   if (BITS (uword) == 32)
1446     {
1447       sum0 =
1448         ip_csum_with_carry (sum0,
1449                             clib_mem_unaligned (&ip0->src_address, u32));
1450       sum0 =
1451         ip_csum_with_carry (sum0,
1452                             clib_mem_unaligned (&ip0->dst_address, u32));
1453     }
1454   else
1455     sum0 =
1456       ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1457
1458   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1459   data_this_buffer = (void *) ip0 + ip_header_length;
1460   if (n_this_buffer + ip_header_length > p0->current_length)
1461     n_this_buffer =
1462       p0->current_length >
1463       ip_header_length ? p0->current_length - ip_header_length : 0;
1464   while (1)
1465     {
1466       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1467       n_bytes_left -= n_this_buffer;
1468       if (n_bytes_left == 0)
1469         break;
1470
1471       ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1472       p0 = vlib_get_buffer (vm, p0->next_buffer);
1473       data_this_buffer = vlib_buffer_get_current (p0);
1474       n_this_buffer = p0->current_length;
1475     }
1476
1477   sum16 = ~ip_csum_fold (sum0);
1478
1479   return sum16;
1480 }
1481
1482 u32
1483 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1484 {
1485   ip4_header_t *ip0 = vlib_buffer_get_current (p0);
1486   udp_header_t *udp0;
1487   u16 sum16;
1488
1489   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1490           || ip0->protocol == IP_PROTOCOL_UDP);
1491
1492   udp0 = (void *) (ip0 + 1);
1493   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1494     {
1495       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1496                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1497       return p0->flags;
1498     }
1499
1500   sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1501
1502   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1503                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1504
1505   return p0->flags;
1506 }
1507
1508 /* *INDENT-OFF* */
1509 VNET_FEATURE_ARC_INIT (ip4_local) =
1510 {
1511   .arc_name  = "ip4-local",
1512   .start_nodes = VNET_FEATURES ("ip4-local"),
1513 };
1514 /* *INDENT-ON* */
1515
1516 static inline uword
1517 ip4_local_inline (vlib_main_t * vm,
1518                   vlib_node_runtime_t * node,
1519                   vlib_frame_t * frame, int head_of_feature_arc)
1520 {
1521   ip4_main_t *im = &ip4_main;
1522   ip_lookup_main_t *lm = &im->lookup_main;
1523   ip_local_next_t next_index;
1524   u32 *from, *to_next, n_left_from, n_left_to_next;
1525   vlib_node_runtime_t *error_node =
1526     vlib_node_get_runtime (vm, ip4_input_node.index);
1527   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1528
1529   from = vlib_frame_vector_args (frame);
1530   n_left_from = frame->n_vectors;
1531   next_index = node->cached_next_index;
1532
1533   if (node->flags & VLIB_NODE_FLAG_TRACE)
1534     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1535
1536   while (n_left_from > 0)
1537     {
1538       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1539
1540       while (n_left_from >= 4 && n_left_to_next >= 2)
1541         {
1542           vlib_buffer_t *p0, *p1;
1543           ip4_header_t *ip0, *ip1;
1544           udp_header_t *udp0, *udp1;
1545           ip4_fib_mtrie_t *mtrie0, *mtrie1;
1546           ip4_fib_mtrie_leaf_t leaf0, leaf1;
1547           const dpo_id_t *dpo0, *dpo1;
1548           const load_balance_t *lb0, *lb1;
1549           u32 pi0, ip_len0, udp_len0, flags0, next0, fib_index0, lbi0;
1550           u32 pi1, ip_len1, udp_len1, flags1, next1, fib_index1, lbi1;
1551           i32 len_diff0, len_diff1;
1552           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1553           u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1554           u32 sw_if_index0, sw_if_index1;
1555
1556           pi0 = to_next[0] = from[0];
1557           pi1 = to_next[1] = from[1];
1558           from += 2;
1559           n_left_from -= 2;
1560           to_next += 2;
1561           n_left_to_next -= 2;
1562
1563           next0 = next1 = IP_LOCAL_NEXT_DROP;
1564
1565           p0 = vlib_get_buffer (vm, pi0);
1566           p1 = vlib_get_buffer (vm, pi1);
1567
1568           ip0 = vlib_buffer_get_current (p0);
1569           ip1 = vlib_buffer_get_current (p1);
1570
1571           vnet_buffer (p0)->ip.start_of_ip_header = p0->current_data;
1572           vnet_buffer (p1)->ip.start_of_ip_header = p1->current_data;
1573
1574           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1575           sw_if_index1 = vnet_buffer (p1)->sw_if_index[VLIB_RX];
1576
1577           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1578           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index1);
1579
1580           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1581           fib_index0 =
1582             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1583              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1584
1585           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index1);
1586           fib_index1 =
1587             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
1588              (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
1589
1590           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1591           mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
1592
1593           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1594           leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, &ip1->src_address);
1595
1596           /* Treat IP frag packets as "experimental" protocol for now
1597              until support of IP frag reassembly is implemented */
1598           proto0 = ip4_is_fragment (ip0) ? 0xfe : ip0->protocol;
1599           proto1 = ip4_is_fragment (ip1) ? 0xfe : ip1->protocol;
1600
1601           if (head_of_feature_arc == 0)
1602             {
1603               error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1604               goto skip_checks;
1605             }
1606
1607           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1608           is_udp1 = proto1 == IP_PROTOCOL_UDP;
1609           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1610           is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1611
1612           flags0 = p0->flags;
1613           flags1 = p1->flags;
1614
1615           good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1616           good_tcp_udp1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1617
1618           udp0 = ip4_next_header (ip0);
1619           udp1 = ip4_next_header (ip1);
1620
1621           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1622           good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1623           good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1624
1625           leaf0 =
1626             ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
1627           leaf1 =
1628             ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 1);
1629
1630           /* Verify UDP length. */
1631           ip_len0 = clib_net_to_host_u16 (ip0->length);
1632           ip_len1 = clib_net_to_host_u16 (ip1->length);
1633           udp_len0 = clib_net_to_host_u16 (udp0->length);
1634           udp_len1 = clib_net_to_host_u16 (udp1->length);
1635
1636           len_diff0 = ip_len0 - udp_len0;
1637           len_diff1 = ip_len1 - udp_len1;
1638
1639           len_diff0 = is_udp0 ? len_diff0 : 0;
1640           len_diff1 = is_udp1 ? len_diff1 : 0;
1641
1642           if (PREDICT_FALSE (!(is_tcp_udp0 & is_tcp_udp1
1643                                & good_tcp_udp0 & good_tcp_udp1)))
1644             {
1645               if (is_tcp_udp0)
1646                 {
1647                   if (is_tcp_udp0
1648                       && !(flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1649                     flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
1650                   good_tcp_udp0 =
1651                     (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1652                   good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1653                 }
1654               if (is_tcp_udp1)
1655                 {
1656                   if (is_tcp_udp1
1657                       && !(flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1658                     flags1 = ip4_tcp_udp_validate_checksum (vm, p1);
1659                   good_tcp_udp1 =
1660                     (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1661                   good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1662                 }
1663             }
1664
1665           good_tcp_udp0 &= len_diff0 >= 0;
1666           good_tcp_udp1 &= len_diff1 >= 0;
1667
1668           leaf0 =
1669             ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1670           leaf1 =
1671             ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 2);
1672
1673           error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1674
1675           error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
1676           error1 = len_diff1 < 0 ? IP4_ERROR_UDP_LENGTH : error1;
1677
1678           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1679           error0 = (is_tcp_udp0 && !good_tcp_udp0
1680                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1681           error1 = (is_tcp_udp1 && !good_tcp_udp1
1682                     ? IP4_ERROR_TCP_CHECKSUM + is_udp1 : error1);
1683
1684           leaf0 =
1685             ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1686           leaf1 =
1687             ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 3);
1688
1689           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0 =
1690             ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1691           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1692
1693           vnet_buffer (p1)->ip.adj_index[VLIB_RX] = lbi1 =
1694             ip4_fib_mtrie_leaf_get_adj_index (leaf1);
1695           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = lbi1;
1696
1697           lb0 = load_balance_get (lbi0);
1698           lb1 = load_balance_get (lbi1);
1699           dpo0 = load_balance_get_bucket_i (lb0, 0);
1700           dpo1 = load_balance_get_bucket_i (lb1, 0);
1701
1702           /*
1703            * Must have a route to source otherwise we drop the packet.
1704            * ip4 broadcasts are accepted, e.g. to make dhcp client work
1705            *
1706            * The checks are:
1707            *  - the source is a recieve => it's from us => bogus, do this
1708            *    first since it sets a different error code.
1709            *  - uRPF check for any route to source - accept if passes.
1710            *  - allow packets destined to the broadcast address from unknown sources
1711            */
1712           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1713                      dpo0->dpoi_type == DPO_RECEIVE) ?
1714                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1715           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1716                      !fib_urpf_check_size (lb0->lb_urpf) &&
1717                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1718                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1719           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1720                      dpo1->dpoi_type == DPO_RECEIVE) ?
1721                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error1);
1722           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1723                      !fib_urpf_check_size (lb1->lb_urpf) &&
1724                      ip1->dst_address.as_u32 != 0xFFFFFFFF)
1725                     ? IP4_ERROR_SRC_LOOKUP_MISS : error1);
1726
1727         skip_checks:
1728
1729           next0 = lm->local_next_by_ip_protocol[proto0];
1730           next1 = lm->local_next_by_ip_protocol[proto1];
1731
1732           next0 =
1733             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1734           next1 =
1735             error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1736
1737           p0->error = error0 ? error_node->errors[error0] : 0;
1738           p1->error = error1 ? error_node->errors[error1] : 0;
1739
1740           if (head_of_feature_arc)
1741             {
1742               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1743                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1744               if (PREDICT_TRUE (error1 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1745                 vnet_feature_arc_start (arc_index, sw_if_index1, &next1, p1);
1746             }
1747
1748           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1749                                            n_left_to_next, pi0, pi1,
1750                                            next0, next1);
1751         }
1752
1753       while (n_left_from > 0 && n_left_to_next > 0)
1754         {
1755           vlib_buffer_t *p0;
1756           ip4_header_t *ip0;
1757           udp_header_t *udp0;
1758           ip4_fib_mtrie_t *mtrie0;
1759           ip4_fib_mtrie_leaf_t leaf0;
1760           u32 pi0, next0, ip_len0, udp_len0, flags0, fib_index0, lbi0;
1761           i32 len_diff0;
1762           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1763           load_balance_t *lb0;
1764           const dpo_id_t *dpo0;
1765           u32 sw_if_index0;
1766
1767           pi0 = to_next[0] = from[0];
1768           from += 1;
1769           n_left_from -= 1;
1770           to_next += 1;
1771           n_left_to_next -= 1;
1772
1773           next0 = IP_LOCAL_NEXT_DROP;
1774
1775           p0 = vlib_get_buffer (vm, pi0);
1776
1777           ip0 = vlib_buffer_get_current (p0);
1778
1779           vnet_buffer (p0)->ip.start_of_ip_header = p0->current_data;
1780
1781           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1782
1783           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1784
1785           fib_index0 =
1786             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1787              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1788
1789           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1790
1791           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1792
1793           /* Treat IP frag packets as "experimental" protocol for now
1794              until support of IP frag reassembly is implemented */
1795           proto0 = ip4_is_fragment (ip0) ? 0xfe : ip0->protocol;
1796
1797           if (head_of_feature_arc == 0)
1798             {
1799               error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
1800               goto skip_check;
1801             }
1802
1803           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1804           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1805
1806           flags0 = p0->flags;
1807
1808           good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1809
1810           udp0 = ip4_next_header (ip0);
1811
1812           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1813           good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1814
1815           leaf0 =
1816             ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
1817
1818           /* Verify UDP length. */
1819           ip_len0 = clib_net_to_host_u16 (ip0->length);
1820           udp_len0 = clib_net_to_host_u16 (udp0->length);
1821
1822           len_diff0 = ip_len0 - udp_len0;
1823
1824           len_diff0 = is_udp0 ? len_diff0 : 0;
1825
1826           if (PREDICT_FALSE (!(is_tcp_udp0 & good_tcp_udp0)))
1827             {
1828               if (is_tcp_udp0)
1829                 {
1830                   if (is_tcp_udp0
1831                       && !(flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1832                     flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
1833                   good_tcp_udp0 =
1834                     (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1835                   good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1836                 }
1837             }
1838
1839           good_tcp_udp0 &= len_diff0 >= 0;
1840
1841           leaf0 =
1842             ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1843
1844           error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
1845
1846           error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
1847
1848           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1849           error0 = (is_tcp_udp0 && !good_tcp_udp0
1850                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1851
1852           leaf0 =
1853             ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1854
1855           lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1856           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1857
1858           lb0 = load_balance_get (lbi0);
1859           dpo0 = load_balance_get_bucket_i (lb0, 0);
1860
1861           vnet_buffer (p0)->ip.adj_index[VLIB_TX] =
1862             vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0;
1863
1864           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1865                      dpo0->dpoi_type == DPO_RECEIVE) ?
1866                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1867           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1868                      !fib_urpf_check_size (lb0->lb_urpf) &&
1869                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1870                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1871
1872         skip_check:
1873
1874           next0 = lm->local_next_by_ip_protocol[proto0];
1875
1876           next0 =
1877             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1878
1879           p0->error = error0 ? error_node->errors[error0] : 0;
1880
1881           if (head_of_feature_arc)
1882             {
1883               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1884                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1885             }
1886
1887           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1888                                            n_left_to_next, pi0, next0);
1889
1890         }
1891
1892       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1893     }
1894
1895   return frame->n_vectors;
1896 }
1897
1898 static uword
1899 ip4_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1900 {
1901   return ip4_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1902 }
1903
1904 /* *INDENT-OFF* */
1905 VLIB_REGISTER_NODE (ip4_local_node) =
1906 {
1907   .function = ip4_local,
1908   .name = "ip4-local",
1909   .vector_size = sizeof (u32),
1910   .format_trace = format_ip4_forward_next_trace,
1911   .n_next_nodes = IP_LOCAL_N_NEXT,
1912   .next_nodes =
1913   {
1914     [IP_LOCAL_NEXT_DROP] = "error-drop",
1915     [IP_LOCAL_NEXT_PUNT] = "error-punt",
1916     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
1917     [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",},
1918 };
1919 /* *INDENT-ON* */
1920
1921 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local);
1922
1923 static uword
1924 ip4_local_end_of_arc (vlib_main_t * vm,
1925                       vlib_node_runtime_t * node, vlib_frame_t * frame)
1926 {
1927   return ip4_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1928 }
1929
1930 /* *INDENT-OFF* */
1931 VLIB_REGISTER_NODE (ip4_local_end_of_arc_node,static) = {
1932   .function = ip4_local_end_of_arc,
1933   .name = "ip4-local-end-of-arc",
1934   .vector_size = sizeof (u32),
1935
1936   .format_trace = format_ip4_forward_next_trace,
1937   .sibling_of = "ip4-local",
1938 };
1939
1940 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_end_of_arc_node, ip4_local_end_of_arc)
1941
1942 VNET_FEATURE_INIT (ip4_local_end_of_arc, static) = {
1943   .arc_name = "ip4-local",
1944   .node_name = "ip4-local-end-of-arc",
1945   .runs_before = 0, /* not before any other features */
1946 };
1947 /* *INDENT-ON* */
1948
1949 void
1950 ip4_register_protocol (u32 protocol, u32 node_index)
1951 {
1952   vlib_main_t *vm = vlib_get_main ();
1953   ip4_main_t *im = &ip4_main;
1954   ip_lookup_main_t *lm = &im->lookup_main;
1955
1956   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1957   lm->local_next_by_ip_protocol[protocol] =
1958     vlib_node_add_next (vm, ip4_local_node.index, node_index);
1959 }
1960
1961 static clib_error_t *
1962 show_ip_local_command_fn (vlib_main_t * vm,
1963                           unformat_input_t * input, vlib_cli_command_t * cmd)
1964 {
1965   ip4_main_t *im = &ip4_main;
1966   ip_lookup_main_t *lm = &im->lookup_main;
1967   int i;
1968
1969   vlib_cli_output (vm, "Protocols handled by ip4_local");
1970   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
1971     {
1972       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
1973         vlib_cli_output (vm, "%d", i);
1974     }
1975   return 0;
1976 }
1977
1978
1979
1980 /*?
1981  * Display the set of protocols handled by the local IPv4 stack.
1982  *
1983  * @cliexpar
1984  * Example of how to display local protocol table:
1985  * @cliexstart{show ip local}
1986  * Protocols handled by ip4_local
1987  * 1
1988  * 17
1989  * 47
1990  * @cliexend
1991 ?*/
1992 /* *INDENT-OFF* */
1993 VLIB_CLI_COMMAND (show_ip_local, static) =
1994 {
1995   .path = "show ip local",
1996   .function = show_ip_local_command_fn,
1997   .short_help = "show ip local",
1998 };
1999 /* *INDENT-ON* */
2000
2001 always_inline uword
2002 ip4_arp_inline (vlib_main_t * vm,
2003                 vlib_node_runtime_t * node,
2004                 vlib_frame_t * frame, int is_glean)
2005 {
2006   vnet_main_t *vnm = vnet_get_main ();
2007   ip4_main_t *im = &ip4_main;
2008   ip_lookup_main_t *lm = &im->lookup_main;
2009   u32 *from, *to_next_drop;
2010   uword n_left_from, n_left_to_next_drop, next_index;
2011   static f64 time_last_seed_change = -1e100;
2012   static u32 hash_seeds[3];
2013   static uword hash_bitmap[256 / BITS (uword)];
2014   f64 time_now;
2015
2016   if (node->flags & VLIB_NODE_FLAG_TRACE)
2017     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2018
2019   time_now = vlib_time_now (vm);
2020   if (time_now - time_last_seed_change > 1e-3)
2021     {
2022       uword i;
2023       u32 *r = clib_random_buffer_get_data (&vm->random_buffer,
2024                                             sizeof (hash_seeds));
2025       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
2026         hash_seeds[i] = r[i];
2027
2028       /* Mark all hash keys as been no-seen before. */
2029       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
2030         hash_bitmap[i] = 0;
2031
2032       time_last_seed_change = time_now;
2033     }
2034
2035   from = vlib_frame_vector_args (frame);
2036   n_left_from = frame->n_vectors;
2037   next_index = node->cached_next_index;
2038   if (next_index == IP4_ARP_NEXT_DROP)
2039     next_index = IP4_ARP_N_NEXT;        /* point to first interface */
2040
2041   while (n_left_from > 0)
2042     {
2043       vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
2044                            to_next_drop, n_left_to_next_drop);
2045
2046       while (n_left_from > 0 && n_left_to_next_drop > 0)
2047         {
2048           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
2049           ip_adjacency_t *adj0;
2050           vlib_buffer_t *p0;
2051           ip4_header_t *ip0;
2052           uword bm0;
2053
2054           pi0 = from[0];
2055
2056           p0 = vlib_get_buffer (vm, pi0);
2057
2058           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2059           adj0 = ip_get_adjacency (lm, adj_index0);
2060           ip0 = vlib_buffer_get_current (p0);
2061
2062           a0 = hash_seeds[0];
2063           b0 = hash_seeds[1];
2064           c0 = hash_seeds[2];
2065
2066           sw_if_index0 = adj0->rewrite_header.sw_if_index;
2067           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
2068
2069           if (is_glean)
2070             {
2071               /*
2072                * this is the Glean case, so we are ARPing for the
2073                * packet's destination
2074                */
2075               a0 ^= ip0->dst_address.data_u32;
2076             }
2077           else
2078             {
2079               a0 ^= adj0->sub_type.nbr.next_hop.ip4.data_u32;
2080             }
2081           b0 ^= sw_if_index0;
2082
2083           hash_v3_finalize32 (a0, b0, c0);
2084
2085           c0 &= BITS (hash_bitmap) - 1;
2086           c0 = c0 / BITS (uword);
2087           m0 = (uword) 1 << (c0 % BITS (uword));
2088
2089           bm0 = hash_bitmap[c0];
2090           drop0 = (bm0 & m0) != 0;
2091
2092           /* Mark it as seen. */
2093           hash_bitmap[c0] = bm0 | m0;
2094
2095           from += 1;
2096           n_left_from -= 1;
2097           to_next_drop[0] = pi0;
2098           to_next_drop += 1;
2099           n_left_to_next_drop -= 1;
2100
2101           p0->error =
2102             node->errors[drop0 ? IP4_ARP_ERROR_DROP :
2103                          IP4_ARP_ERROR_REQUEST_SENT];
2104
2105           /*
2106            * the adj has been updated to a rewrite but the node the DPO that got
2107            * us here hasn't - yet. no big deal. we'll drop while we wait.
2108            */
2109           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
2110             continue;
2111
2112           if (drop0)
2113             continue;
2114
2115           /*
2116            * Can happen if the control-plane is programming tables
2117            * with traffic flowing; at least that's today's lame excuse.
2118            */
2119           if ((is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_GLEAN)
2120               || (!is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP))
2121             {
2122               p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
2123             }
2124           else
2125             /* Send ARP request. */
2126             {
2127               u32 bi0 = 0;
2128               vlib_buffer_t *b0;
2129               ethernet_arp_header_t *h0;
2130               vnet_hw_interface_t *hw_if0;
2131
2132               h0 =
2133                 vlib_packet_template_get_packet (vm,
2134                                                  &im->ip4_arp_request_packet_template,
2135                                                  &bi0);
2136
2137               /* Add rewrite/encap string for ARP packet. */
2138               vnet_rewrite_one_header (adj0[0], h0,
2139                                        sizeof (ethernet_header_t));
2140
2141               hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2142
2143               /* Src ethernet address in ARP header. */
2144               clib_memcpy (h0->ip4_over_ethernet[0].ethernet,
2145                            hw_if0->hw_address,
2146                            sizeof (h0->ip4_over_ethernet[0].ethernet));
2147
2148               if (is_glean)
2149                 {
2150                   /* The interface's source address is stashed in the Glean Adj */
2151                   h0->ip4_over_ethernet[0].ip4 =
2152                     adj0->sub_type.glean.receive_addr.ip4;
2153
2154                   /* Copy in destination address we are requesting. This is the
2155                    * glean case, so it's the packet's destination.*/
2156                   h0->ip4_over_ethernet[1].ip4.data_u32 =
2157                     ip0->dst_address.data_u32;
2158                 }
2159               else
2160                 {
2161                   /* Src IP address in ARP header. */
2162                   if (ip4_src_address_for_packet (lm, sw_if_index0,
2163                                                   &h0->
2164                                                   ip4_over_ethernet[0].ip4))
2165                     {
2166                       /* No source address available */
2167                       p0->error =
2168                         node->errors[IP4_ARP_ERROR_NO_SOURCE_ADDRESS];
2169                       vlib_buffer_free (vm, &bi0, 1);
2170                       continue;
2171                     }
2172
2173                   /* Copy in destination address we are requesting from the
2174                      incomplete adj */
2175                   h0->ip4_over_ethernet[1].ip4.data_u32 =
2176                     adj0->sub_type.nbr.next_hop.ip4.as_u32;
2177                 }
2178
2179               vlib_buffer_copy_trace_flag (vm, p0, bi0);
2180               b0 = vlib_get_buffer (vm, bi0);
2181               vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
2182
2183               vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2184
2185               vlib_set_next_frame_buffer (vm, node,
2186                                           adj0->rewrite_header.next_index,
2187                                           bi0);
2188             }
2189         }
2190
2191       vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
2192     }
2193
2194   return frame->n_vectors;
2195 }
2196
2197 static uword
2198 ip4_arp (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
2199 {
2200   return (ip4_arp_inline (vm, node, frame, 0));
2201 }
2202
2203 static uword
2204 ip4_glean (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
2205 {
2206   return (ip4_arp_inline (vm, node, frame, 1));
2207 }
2208
2209 static char *ip4_arp_error_strings[] = {
2210   [IP4_ARP_ERROR_DROP] = "address overflow drops",
2211   [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
2212   [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
2213   [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
2214   [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
2215   [IP4_ARP_ERROR_NO_SOURCE_ADDRESS] = "no source address for ARP request",
2216 };
2217
2218 VLIB_REGISTER_NODE (ip4_arp_node) =
2219 {
2220   .function = ip4_arp,.name = "ip4-arp",.vector_size =
2221     sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_errors =
2222     ARRAY_LEN (ip4_arp_error_strings),.error_strings =
2223     ip4_arp_error_strings,.n_next_nodes = IP4_ARP_N_NEXT,.next_nodes =
2224   {
2225   [IP4_ARP_NEXT_DROP] = "error-drop",}
2226 ,};
2227
2228 VLIB_REGISTER_NODE (ip4_glean_node) =
2229 {
2230   .function = ip4_glean,.name = "ip4-glean",.vector_size =
2231     sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_errors =
2232     ARRAY_LEN (ip4_arp_error_strings),.error_strings =
2233     ip4_arp_error_strings,.n_next_nodes = IP4_ARP_N_NEXT,.next_nodes =
2234   {
2235   [IP4_ARP_NEXT_DROP] = "error-drop",}
2236 ,};
2237
2238 #define foreach_notrace_ip4_arp_error           \
2239 _(DROP)                                         \
2240 _(REQUEST_SENT)                                 \
2241 _(REPLICATE_DROP)                               \
2242 _(REPLICATE_FAIL)
2243
2244 clib_error_t *
2245 arp_notrace_init (vlib_main_t * vm)
2246 {
2247   vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, ip4_arp_node.index);
2248
2249   /* don't trace ARP request packets */
2250 #define _(a)                                    \
2251     vnet_pcap_drop_trace_filter_add_del         \
2252         (rt->errors[IP4_ARP_ERROR_##a],         \
2253          1 /* is_add */);
2254   foreach_notrace_ip4_arp_error;
2255 #undef _
2256   return 0;
2257 }
2258
2259 VLIB_INIT_FUNCTION (arp_notrace_init);
2260
2261
2262 /* Send an ARP request to see if given destination is reachable on given interface. */
2263 clib_error_t *
2264 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
2265 {
2266   vnet_main_t *vnm = vnet_get_main ();
2267   ip4_main_t *im = &ip4_main;
2268   ethernet_arp_header_t *h;
2269   ip4_address_t *src;
2270   ip_interface_address_t *ia;
2271   ip_adjacency_t *adj;
2272   vnet_hw_interface_t *hi;
2273   vnet_sw_interface_t *si;
2274   vlib_buffer_t *b;
2275   u32 bi = 0;
2276
2277   si = vnet_get_sw_interface (vnm, sw_if_index);
2278
2279   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2280     {
2281       return clib_error_return (0, "%U: interface %U down",
2282                                 format_ip4_address, dst,
2283                                 format_vnet_sw_if_index_name, vnm,
2284                                 sw_if_index);
2285     }
2286
2287   src =
2288     ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2289   if (!src)
2290     {
2291       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2292       return clib_error_return
2293         (0,
2294          "no matching interface address for destination %U (interface %U)",
2295          format_ip4_address, dst, format_vnet_sw_if_index_name, vnm,
2296          sw_if_index);
2297     }
2298
2299   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
2300
2301   h =
2302     vlib_packet_template_get_packet (vm,
2303                                      &im->ip4_arp_request_packet_template,
2304                                      &bi);
2305
2306   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2307
2308   clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address,
2309                sizeof (h->ip4_over_ethernet[0].ethernet));
2310
2311   h->ip4_over_ethernet[0].ip4 = src[0];
2312   h->ip4_over_ethernet[1].ip4 = dst[0];
2313
2314   b = vlib_get_buffer (vm, bi);
2315   vnet_buffer (b)->sw_if_index[VLIB_RX] =
2316     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2317
2318   /* Add encapsulation string for software interface (e.g. ethernet header). */
2319   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2320   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2321
2322   {
2323     vlib_frame_t *f = vlib_get_frame_to_node (vm, hi->output_node_index);
2324     u32 *to_next = vlib_frame_vector_args (f);
2325     to_next[0] = bi;
2326     f->n_vectors = 1;
2327     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2328   }
2329
2330   return /* no error */ 0;
2331 }
2332
2333 typedef enum
2334 {
2335   IP4_REWRITE_NEXT_DROP,
2336   IP4_REWRITE_NEXT_ICMP_ERROR,
2337 } ip4_rewrite_next_t;
2338
2339 always_inline uword
2340 ip4_rewrite_inline (vlib_main_t * vm,
2341                     vlib_node_runtime_t * node,
2342                     vlib_frame_t * frame,
2343                     int do_counters, int is_midchain, int is_mcast)
2344 {
2345   ip_lookup_main_t *lm = &ip4_main.lookup_main;
2346   u32 *from = vlib_frame_vector_args (frame);
2347   u32 n_left_from, n_left_to_next, *to_next, next_index;
2348   vlib_node_runtime_t *error_node =
2349     vlib_node_get_runtime (vm, ip4_input_node.index);
2350
2351   n_left_from = frame->n_vectors;
2352   next_index = node->cached_next_index;
2353   u32 cpu_index = os_get_cpu_number ();
2354
2355   while (n_left_from > 0)
2356     {
2357       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2358
2359       while (n_left_from >= 4 && n_left_to_next >= 2)
2360         {
2361           ip_adjacency_t *adj0, *adj1;
2362           vlib_buffer_t *p0, *p1;
2363           ip4_header_t *ip0, *ip1;
2364           u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
2365           u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
2366           u32 tx_sw_if_index0, tx_sw_if_index1;
2367
2368           /* Prefetch next iteration. */
2369           {
2370             vlib_buffer_t *p2, *p3;
2371
2372             p2 = vlib_get_buffer (vm, from[2]);
2373             p3 = vlib_get_buffer (vm, from[3]);
2374
2375             vlib_prefetch_buffer_header (p2, STORE);
2376             vlib_prefetch_buffer_header (p3, STORE);
2377
2378             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2379             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2380           }
2381
2382           pi0 = to_next[0] = from[0];
2383           pi1 = to_next[1] = from[1];
2384
2385           from += 2;
2386           n_left_from -= 2;
2387           to_next += 2;
2388           n_left_to_next -= 2;
2389
2390           p0 = vlib_get_buffer (vm, pi0);
2391           p1 = vlib_get_buffer (vm, pi1);
2392
2393           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2394           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
2395
2396           /*
2397            * pre-fetch the per-adjacency counters
2398            */
2399           if (do_counters)
2400             {
2401               vlib_prefetch_combined_counter (&adjacency_counters,
2402                                               cpu_index, adj_index0);
2403               vlib_prefetch_combined_counter (&adjacency_counters,
2404                                               cpu_index, adj_index1);
2405             }
2406
2407           ip0 = vlib_buffer_get_current (p0);
2408           ip1 = vlib_buffer_get_current (p1);
2409
2410           error0 = error1 = IP4_ERROR_NONE;
2411           next0 = next1 = IP4_REWRITE_NEXT_DROP;
2412
2413           /* Decrement TTL & update checksum.
2414              Works either endian, so no need for byte swap. */
2415           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_LOCALLY_ORIGINATED)))
2416             {
2417               i32 ttl0 = ip0->ttl;
2418
2419               /* Input node should have reject packets with ttl 0. */
2420               ASSERT (ip0->ttl > 0);
2421
2422               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2423               checksum0 += checksum0 >= 0xffff;
2424
2425               ip0->checksum = checksum0;
2426               ttl0 -= 1;
2427               ip0->ttl = ttl0;
2428
2429               /*
2430                * If the ttl drops below 1 when forwarding, generate
2431                * an ICMP response.
2432                */
2433               if (PREDICT_FALSE (ttl0 <= 0))
2434                 {
2435                   error0 = IP4_ERROR_TIME_EXPIRED;
2436                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2437                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2438                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2439                                                0);
2440                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2441                 }
2442
2443               /* Verify checksum. */
2444               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2445             }
2446           else
2447             {
2448               p0->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
2449             }
2450           if (PREDICT_TRUE (!(p1->flags & VNET_BUFFER_LOCALLY_ORIGINATED)))
2451             {
2452               i32 ttl1 = ip1->ttl;
2453
2454               /* Input node should have reject packets with ttl 0. */
2455               ASSERT (ip1->ttl > 0);
2456
2457               checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2458               checksum1 += checksum1 >= 0xffff;
2459
2460               ip1->checksum = checksum1;
2461               ttl1 -= 1;
2462               ip1->ttl = ttl1;
2463
2464               /*
2465                * If the ttl drops below 1 when forwarding, generate
2466                * an ICMP response.
2467                */
2468               if (PREDICT_FALSE (ttl1 <= 0))
2469                 {
2470                   error1 = IP4_ERROR_TIME_EXPIRED;
2471                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2472                   icmp4_error_set_vnet_buffer (p1, ICMP4_time_exceeded,
2473                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2474                                                0);
2475                   next1 = IP4_REWRITE_NEXT_ICMP_ERROR;
2476                 }
2477
2478               /* Verify checksum. */
2479               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2480               ASSERT (ip1->checksum == ip4_header_checksum (ip1));
2481             }
2482           else
2483             {
2484               p1->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
2485             }
2486
2487           /* Rewrite packet header and updates lengths. */
2488           adj0 = ip_get_adjacency (lm, adj_index0);
2489           adj1 = ip_get_adjacency (lm, adj_index1);
2490
2491           /* Worth pipelining. No guarantee that adj0,1 are hot... */
2492           rw_len0 = adj0[0].rewrite_header.data_bytes;
2493           rw_len1 = adj1[0].rewrite_header.data_bytes;
2494           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2495           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
2496
2497           /* Check MTU of outgoing interface. */
2498           error0 =
2499             (vlib_buffer_length_in_chain (vm, p0) >
2500              adj0[0].
2501              rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
2502              error0);
2503           error1 =
2504             (vlib_buffer_length_in_chain (vm, p1) >
2505              adj1[0].
2506              rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
2507              error1);
2508
2509           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2510            * to see the IP headerr */
2511           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2512             {
2513               next0 = adj0[0].rewrite_header.next_index;
2514               p0->current_data -= rw_len0;
2515               p0->current_length += rw_len0;
2516               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2517               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2518
2519               if (PREDICT_FALSE
2520                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2521                 vnet_feature_arc_start (lm->output_feature_arc_index,
2522                                         tx_sw_if_index0, &next0, p0);
2523             }
2524           if (PREDICT_TRUE (error1 == IP4_ERROR_NONE))
2525             {
2526               next1 = adj1[0].rewrite_header.next_index;
2527               p1->current_data -= rw_len1;
2528               p1->current_length += rw_len1;
2529
2530               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
2531               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
2532
2533               if (PREDICT_FALSE
2534                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2535                 vnet_feature_arc_start (lm->output_feature_arc_index,
2536                                         tx_sw_if_index1, &next1, p1);
2537             }
2538
2539           /* Guess we are only writing on simple Ethernet header. */
2540           vnet_rewrite_two_headers (adj0[0], adj1[0],
2541                                     ip0, ip1, sizeof (ethernet_header_t));
2542
2543           /*
2544            * Bump the per-adjacency counters
2545            */
2546           if (do_counters)
2547             {
2548               vlib_increment_combined_counter
2549                 (&adjacency_counters,
2550                  cpu_index,
2551                  adj_index0, 1,
2552                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2553
2554               vlib_increment_combined_counter
2555                 (&adjacency_counters,
2556                  cpu_index,
2557                  adj_index1, 1,
2558                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
2559             }
2560
2561           if (is_midchain)
2562             {
2563               adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
2564               adj1->sub_type.midchain.fixup_func (vm, adj1, p1);
2565             }
2566           if (is_mcast)
2567             {
2568               /*
2569                * copy bytes from the IP address into the MAC rewrite
2570                */
2571               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2572               vnet_fixup_one_header (adj1[0], &ip1->dst_address, ip1);
2573             }
2574
2575           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2576                                            to_next, n_left_to_next,
2577                                            pi0, pi1, next0, next1);
2578         }
2579
2580       while (n_left_from > 0 && n_left_to_next > 0)
2581         {
2582           ip_adjacency_t *adj0;
2583           vlib_buffer_t *p0;
2584           ip4_header_t *ip0;
2585           u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2586           u32 tx_sw_if_index0;
2587
2588           pi0 = to_next[0] = from[0];
2589
2590           p0 = vlib_get_buffer (vm, pi0);
2591
2592           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2593
2594           adj0 = ip_get_adjacency (lm, adj_index0);
2595
2596           ip0 = vlib_buffer_get_current (p0);
2597
2598           error0 = IP4_ERROR_NONE;
2599           next0 = IP4_REWRITE_NEXT_DROP;        /* drop on error */
2600
2601           /* Decrement TTL & update checksum. */
2602           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_LOCALLY_ORIGINATED)))
2603             {
2604               i32 ttl0 = ip0->ttl;
2605
2606               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2607
2608               checksum0 += checksum0 >= 0xffff;
2609
2610               ip0->checksum = checksum0;
2611
2612               ASSERT (ip0->ttl > 0);
2613
2614               ttl0 -= 1;
2615
2616               ip0->ttl = ttl0;
2617
2618               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2619
2620               if (PREDICT_FALSE (ttl0 <= 0))
2621                 {
2622                   /*
2623                    * If the ttl drops below 1 when forwarding, generate
2624                    * an ICMP response.
2625                    */
2626                   error0 = IP4_ERROR_TIME_EXPIRED;
2627                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2628                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2629                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2630                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2631                                                0);
2632                 }
2633             }
2634           else
2635             {
2636               p0->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
2637             }
2638
2639           if (do_counters)
2640             vlib_prefetch_combined_counter (&adjacency_counters,
2641                                             cpu_index, adj_index0);
2642
2643           /* Guess we are only writing on simple Ethernet header. */
2644           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2645           if (is_mcast)
2646             {
2647               /*
2648                * copy bytes from the IP address into the MAC rewrite
2649                */
2650               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2651             }
2652
2653           /* Update packet buffer attributes/set output interface. */
2654           rw_len0 = adj0[0].rewrite_header.data_bytes;
2655           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2656
2657           if (do_counters)
2658             vlib_increment_combined_counter
2659               (&adjacency_counters,
2660                cpu_index, adj_index0, 1,
2661                vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2662
2663           /* Check MTU of outgoing interface. */
2664           error0 = (vlib_buffer_length_in_chain (vm, p0)
2665                     > adj0[0].rewrite_header.max_l3_packet_bytes
2666                     ? IP4_ERROR_MTU_EXCEEDED : error0);
2667
2668           p0->error = error_node->errors[error0];
2669
2670           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2671            * to see the IP headerr */
2672           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2673             {
2674               p0->current_data -= rw_len0;
2675               p0->current_length += rw_len0;
2676               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2677
2678               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2679               next0 = adj0[0].rewrite_header.next_index;
2680
2681               if (is_midchain)
2682                 {
2683                   adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
2684                 }
2685
2686               if (PREDICT_FALSE
2687                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2688                 vnet_feature_arc_start (lm->output_feature_arc_index,
2689                                         tx_sw_if_index0, &next0, p0);
2690
2691             }
2692
2693           from += 1;
2694           n_left_from -= 1;
2695           to_next += 1;
2696           n_left_to_next -= 1;
2697
2698           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2699                                            to_next, n_left_to_next,
2700                                            pi0, next0);
2701         }
2702
2703       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2704     }
2705
2706   /* Need to do trace after rewrites to pick up new packet data. */
2707   if (node->flags & VLIB_NODE_FLAG_TRACE)
2708     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2709
2710   return frame->n_vectors;
2711 }
2712
2713
2714 /** @brief IPv4 rewrite node.
2715     @node ip4-rewrite
2716
2717     This is the IPv4 transit-rewrite node: decrement TTL, fix the ipv4
2718     header checksum, fetch the ip adjacency, check the outbound mtu,
2719     apply the adjacency rewrite, and send pkts to the adjacency
2720     rewrite header's rewrite_next_index.
2721
2722     @param vm vlib_main_t corresponding to the current thread
2723     @param node vlib_node_runtime_t
2724     @param frame vlib_frame_t whose contents should be dispatched
2725
2726     @par Graph mechanics: buffer metadata, next index usage
2727
2728     @em Uses:
2729     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
2730         - the rewrite adjacency index
2731     - <code>adj->lookup_next_index</code>
2732         - Must be IP_LOOKUP_NEXT_REWRITE or IP_LOOKUP_NEXT_ARP, otherwise
2733           the packet will be dropped.
2734     - <code>adj->rewrite_header</code>
2735         - Rewrite string length, rewrite string, next_index
2736
2737     @em Sets:
2738     - <code>b->current_data, b->current_length</code>
2739         - Updated net of applying the rewrite string
2740
2741     <em>Next Indices:</em>
2742     - <code> adj->rewrite_header.next_index </code>
2743       or @c error-drop
2744 */
2745 static uword
2746 ip4_rewrite (vlib_main_t * vm,
2747              vlib_node_runtime_t * node, vlib_frame_t * frame)
2748 {
2749   if (adj_are_counters_enabled ())
2750     return ip4_rewrite_inline (vm, node, frame, 1, 0, 0);
2751   else
2752     return ip4_rewrite_inline (vm, node, frame, 0, 0, 0);
2753 }
2754
2755 static uword
2756 ip4_midchain (vlib_main_t * vm,
2757               vlib_node_runtime_t * node, vlib_frame_t * frame)
2758 {
2759   if (adj_are_counters_enabled ())
2760     return ip4_rewrite_inline (vm, node, frame, 1, 1, 0);
2761   else
2762     return ip4_rewrite_inline (vm, node, frame, 0, 1, 0);
2763 }
2764
2765 static uword
2766 ip4_rewrite_mcast (vlib_main_t * vm,
2767                    vlib_node_runtime_t * node, vlib_frame_t * frame)
2768 {
2769   if (adj_are_counters_enabled ())
2770     return ip4_rewrite_inline (vm, node, frame, 1, 0, 1);
2771   else
2772     return ip4_rewrite_inline (vm, node, frame, 0, 0, 1);
2773 }
2774
2775 /* *INDENT-OFF* */
2776 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2777   .function = ip4_rewrite,
2778   .name = "ip4-rewrite",
2779   .vector_size = sizeof (u32),
2780
2781   .format_trace = format_ip4_rewrite_trace,
2782
2783   .n_next_nodes = 2,
2784   .next_nodes = {
2785     [IP4_REWRITE_NEXT_DROP] = "error-drop",
2786     [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2787   },
2788 };
2789 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite)
2790
2791 VLIB_REGISTER_NODE (ip4_rewrite_mcast_node) = {
2792   .function = ip4_rewrite_mcast,
2793   .name = "ip4-rewrite-mcast",
2794   .vector_size = sizeof (u32),
2795
2796   .format_trace = format_ip4_rewrite_trace,
2797   .sibling_of = "ip4-rewrite",
2798 };
2799 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_mcast_node, ip4_rewrite_mcast)
2800
2801 VLIB_REGISTER_NODE (ip4_midchain_node) = {
2802   .function = ip4_midchain,
2803   .name = "ip4-midchain",
2804   .vector_size = sizeof (u32),
2805   .format_trace = format_ip4_forward_next_trace,
2806   .sibling_of =  "ip4-rewrite",
2807 };
2808 VLIB_NODE_FUNCTION_MULTIARCH (ip4_midchain_node, ip4_midchain);
2809 /* *INDENT-ON */
2810
2811 static clib_error_t *
2812 add_del_interface_table (vlib_main_t * vm,
2813                          unformat_input_t * input, vlib_cli_command_t * cmd)
2814 {
2815   vnet_main_t *vnm = vnet_get_main ();
2816   ip_interface_address_t *ia;
2817   clib_error_t *error = 0;
2818   u32 sw_if_index, table_id;
2819
2820   sw_if_index = ~0;
2821
2822   if (!unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2823     {
2824       error = clib_error_return (0, "unknown interface `%U'",
2825                                  format_unformat_error, input);
2826       goto done;
2827     }
2828
2829   if (unformat (input, "%d", &table_id))
2830     ;
2831   else
2832     {
2833       error = clib_error_return (0, "expected table id `%U'",
2834                                  format_unformat_error, input);
2835       goto done;
2836     }
2837
2838   /*
2839    * If the interface already has in IP address, then a change int
2840    * VRF is not allowed. The IP address applied must first be removed.
2841    * We do not do that automatically here, since VPP has no knowledge
2842    * of whether thoses subnets are valid in the destination VRF.
2843    */
2844   /* *INDENT-OFF* */
2845   foreach_ip_interface_address (&ip4_main.lookup_main,
2846                                 ia, sw_if_index,
2847                                 1 /* honor unnumbered */,
2848   ({
2849       ip4_address_t * a;
2850
2851       a = ip_interface_address_get_address (&ip4_main.lookup_main, ia);
2852       error = clib_error_return (0, "interface %U has address %U",
2853                                  format_vnet_sw_if_index_name, vnm,
2854                                  sw_if_index,
2855                                  format_ip4_address, a);
2856       goto done;
2857    }));
2858    /* *INDENT-ON* */
2859
2860 {
2861   ip4_main_t *im = &ip4_main;
2862   u32 fib_index;
2863
2864   fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, table_id);
2865
2866   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
2867   im->fib_index_by_sw_if_index[sw_if_index] = fib_index;
2868
2869   fib_index = mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, table_id);
2870   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
2871   im->mfib_index_by_sw_if_index[sw_if_index] = fib_index;
2872 }
2873
2874 done:
2875 return error;
2876 }
2877
2878 /*?
2879  * Place the indicated interface into the supplied IPv4 FIB table (also known
2880  * as a VRF). If the FIB table does not exist, this command creates it. To
2881  * display the current IPv4 FIB table, use the command '<em>show ip fib</em>'.
2882  * FIB table will only be displayed if a route has been added to the table, or
2883  * an IP Address is assigned to an interface in the table (which adds a route
2884  * automatically).
2885  *
2886  * @note IP addresses added after setting the interface IP table are added to
2887  * the indicated FIB table. If an IP address is added prior to changing the
2888  * table then this is an error. The control plane must remove these addresses
2889  * first and then change the table. VPP will not automatically move the
2890  * addresses from the old to the new table as it does not know the validity
2891  * of such a change.
2892  *
2893  * @cliexpar
2894  * Example of how to add an interface to an IPv4 FIB table (where 2 is the table-id):
2895  * @cliexcmd{set interface ip table GigabitEthernet2/0/0 2}
2896  ?*/
2897 /* *INDENT-OFF* */
2898 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) =
2899 {
2900   .path = "set interface ip table",
2901   .function = add_del_interface_table,
2902   .short_help = "set interface ip table <interface> <table-id>",
2903 };
2904 /* *INDENT-ON* */
2905
2906 int
2907 ip4_lookup_validate (ip4_address_t * a, u32 fib_index0)
2908 {
2909   ip4_fib_mtrie_t *mtrie0;
2910   ip4_fib_mtrie_leaf_t leaf0;
2911   u32 lbi0;
2912
2913   mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
2914
2915   leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, a);
2916   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 1);
2917   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
2918   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
2919
2920   lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2921
2922   return lbi0 == ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0), a);
2923 }
2924
2925 static clib_error_t *
2926 test_lookup_command_fn (vlib_main_t * vm,
2927                         unformat_input_t * input, vlib_cli_command_t * cmd)
2928 {
2929   ip4_fib_t *fib;
2930   u32 table_id = 0;
2931   f64 count = 1;
2932   u32 n;
2933   int i;
2934   ip4_address_t ip4_base_address;
2935   u64 errors = 0;
2936
2937   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2938     {
2939       if (unformat (input, "table %d", &table_id))
2940         {
2941           /* Make sure the entry exists. */
2942           fib = ip4_fib_get (table_id);
2943           if ((fib) && (fib->index != table_id))
2944             return clib_error_return (0, "<fib-index> %d does not exist",
2945                                       table_id);
2946         }
2947       else if (unformat (input, "count %f", &count))
2948         ;
2949
2950       else if (unformat (input, "%U",
2951                          unformat_ip4_address, &ip4_base_address))
2952         ;
2953       else
2954         return clib_error_return (0, "unknown input `%U'",
2955                                   format_unformat_error, input);
2956     }
2957
2958   n = count;
2959
2960   for (i = 0; i < n; i++)
2961     {
2962       if (!ip4_lookup_validate (&ip4_base_address, table_id))
2963         errors++;
2964
2965       ip4_base_address.as_u32 =
2966         clib_host_to_net_u32 (1 +
2967                               clib_net_to_host_u32 (ip4_base_address.as_u32));
2968     }
2969
2970   if (errors)
2971     vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
2972   else
2973     vlib_cli_output (vm, "No errors in %d lookups\n", n);
2974
2975   return 0;
2976 }
2977
2978 /*?
2979  * Perform a lookup of an IPv4 Address (or range of addresses) in the
2980  * given FIB table to determine if there is a conflict with the
2981  * adjacency table. The fib-id can be determined by using the
2982  * '<em>show ip fib</em>' command. If fib-id is not entered, default value
2983  * of 0 is used.
2984  *
2985  * @todo This command uses fib-id, other commands use table-id (not
2986  * just a name, they are different indexes). Would like to change this
2987  * to table-id for consistency.
2988  *
2989  * @cliexpar
2990  * Example of how to run the test lookup command:
2991  * @cliexstart{test lookup 172.16.1.1 table 1 count 2}
2992  * No errors in 2 lookups
2993  * @cliexend
2994 ?*/
2995 /* *INDENT-OFF* */
2996 VLIB_CLI_COMMAND (lookup_test_command, static) =
2997 {
2998   .path = "test lookup",
2999   .short_help = "test lookup <ipv4-addr> [table <fib-id>] [count <nn>]",
3000   .function = test_lookup_command_fn,
3001 };
3002 /* *INDENT-ON* */
3003
3004 int
3005 vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
3006 {
3007   ip4_main_t *im4 = &ip4_main;
3008   ip4_fib_t *fib;
3009   uword *p = hash_get (im4->fib_index_by_table_id, table_id);
3010
3011   if (p == 0)
3012     return VNET_API_ERROR_NO_SUCH_FIB;
3013
3014   fib = ip4_fib_get (p[0]);
3015
3016   fib->flow_hash_config = flow_hash_config;
3017   return 0;
3018 }
3019
3020 static clib_error_t *
3021 set_ip_flow_hash_command_fn (vlib_main_t * vm,
3022                              unformat_input_t * input,
3023                              vlib_cli_command_t * cmd)
3024 {
3025   int matched = 0;
3026   u32 table_id = 0;
3027   u32 flow_hash_config = 0;
3028   int rv;
3029
3030   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3031     {
3032       if (unformat (input, "table %d", &table_id))
3033         matched = 1;
3034 #define _(a,v) \
3035     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
3036       foreach_flow_hash_bit
3037 #undef _
3038         else
3039         break;
3040     }
3041
3042   if (matched == 0)
3043     return clib_error_return (0, "unknown input `%U'",
3044                               format_unformat_error, input);
3045
3046   rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
3047   switch (rv)
3048     {
3049     case 0:
3050       break;
3051
3052     case VNET_API_ERROR_NO_SUCH_FIB:
3053       return clib_error_return (0, "no such FIB table %d", table_id);
3054
3055     default:
3056       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3057       break;
3058     }
3059
3060   return 0;
3061 }
3062
3063 /*?
3064  * Configure the set of IPv4 fields used by the flow hash.
3065  *
3066  * @cliexpar
3067  * Example of how to set the flow hash on a given table:
3068  * @cliexcmd{set ip flow-hash table 7 dst sport dport proto}
3069  * Example of display the configured flow hash:
3070  * @cliexstart{show ip fib}
3071  * ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
3072  * 0.0.0.0/0
3073  *   unicast-ip4-chain
3074  *   [@0]: dpo-load-balance: [index:0 buckets:1 uRPF:0 to:[0:0]]
3075  *     [0] [@0]: dpo-drop ip6
3076  * 0.0.0.0/32
3077  *   unicast-ip4-chain
3078  *   [@0]: dpo-load-balance: [index:1 buckets:1 uRPF:1 to:[0:0]]
3079  *     [0] [@0]: dpo-drop ip6
3080  * 224.0.0.0/8
3081  *   unicast-ip4-chain
3082  *   [@0]: dpo-load-balance: [index:3 buckets:1 uRPF:3 to:[0:0]]
3083  *     [0] [@0]: dpo-drop ip6
3084  * 6.0.1.2/32
3085  *   unicast-ip4-chain
3086  *   [@0]: dpo-load-balance: [index:30 buckets:1 uRPF:29 to:[0:0]]
3087  *     [0] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
3088  * 7.0.0.1/32
3089  *   unicast-ip4-chain
3090  *   [@0]: dpo-load-balance: [index:31 buckets:4 uRPF:30 to:[0:0]]
3091  *     [0] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3092  *     [1] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3093  *     [2] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3094  *     [3] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
3095  * 240.0.0.0/8
3096  *   unicast-ip4-chain
3097  *   [@0]: dpo-load-balance: [index:2 buckets:1 uRPF:2 to:[0:0]]
3098  *     [0] [@0]: dpo-drop ip6
3099  * 255.255.255.255/32
3100  *   unicast-ip4-chain
3101  *   [@0]: dpo-load-balance: [index:4 buckets:1 uRPF:4 to:[0:0]]
3102  *     [0] [@0]: dpo-drop ip6
3103  * ipv4-VRF:7, fib_index 1, flow hash: dst sport dport proto
3104  * 0.0.0.0/0
3105  *   unicast-ip4-chain
3106  *   [@0]: dpo-load-balance: [index:12 buckets:1 uRPF:11 to:[0:0]]
3107  *     [0] [@0]: dpo-drop ip6
3108  * 0.0.0.0/32
3109  *   unicast-ip4-chain
3110  *   [@0]: dpo-load-balance: [index:13 buckets:1 uRPF:12 to:[0:0]]
3111  *     [0] [@0]: dpo-drop ip6
3112  * 172.16.1.0/24
3113  *   unicast-ip4-chain
3114  *   [@0]: dpo-load-balance: [index:17 buckets:1 uRPF:16 to:[0:0]]
3115  *     [0] [@4]: ipv4-glean: af_packet0
3116  * 172.16.1.1/32
3117  *   unicast-ip4-chain
3118  *   [@0]: dpo-load-balance: [index:18 buckets:1 uRPF:17 to:[1:84]]
3119  *     [0] [@2]: dpo-receive: 172.16.1.1 on af_packet0
3120  * 172.16.1.2/32
3121  *   unicast-ip4-chain
3122  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
3123  *     [0] [@5]: ipv4 via 172.16.1.2 af_packet0: IP4: 02:fe:9e:70:7a:2b -> 26:a5:f6:9c:3a:36
3124  * 172.16.2.0/24
3125  *   unicast-ip4-chain
3126  *   [@0]: dpo-load-balance: [index:19 buckets:1 uRPF:18 to:[0:0]]
3127  *     [0] [@4]: ipv4-glean: af_packet1
3128  * 172.16.2.1/32
3129  *   unicast-ip4-chain
3130  *   [@0]: dpo-load-balance: [index:20 buckets:1 uRPF:19 to:[0:0]]
3131  *     [0] [@2]: dpo-receive: 172.16.2.1 on af_packet1
3132  * 224.0.0.0/8
3133  *   unicast-ip4-chain
3134  *   [@0]: dpo-load-balance: [index:15 buckets:1 uRPF:14 to:[0:0]]
3135  *     [0] [@0]: dpo-drop ip6
3136  * 240.0.0.0/8
3137  *   unicast-ip4-chain
3138  *   [@0]: dpo-load-balance: [index:14 buckets:1 uRPF:13 to:[0:0]]
3139  *     [0] [@0]: dpo-drop ip6
3140  * 255.255.255.255/32
3141  *   unicast-ip4-chain
3142  *   [@0]: dpo-load-balance: [index:16 buckets:1 uRPF:15 to:[0:0]]
3143  *     [0] [@0]: dpo-drop ip6
3144  * @cliexend
3145 ?*/
3146 /* *INDENT-OFF* */
3147 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) =
3148 {
3149   .path = "set ip flow-hash",
3150   .short_help =
3151   "set ip flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
3152   .function = set_ip_flow_hash_command_fn,
3153 };
3154 /* *INDENT-ON* */
3155
3156 int
3157 vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3158                              u32 table_index)
3159 {
3160   vnet_main_t *vnm = vnet_get_main ();
3161   vnet_interface_main_t *im = &vnm->interface_main;
3162   ip4_main_t *ipm = &ip4_main;
3163   ip_lookup_main_t *lm = &ipm->lookup_main;
3164   vnet_classify_main_t *cm = &vnet_classify_main;
3165   ip4_address_t *if_addr;
3166
3167   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3168     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3169
3170   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3171     return VNET_API_ERROR_NO_SUCH_ENTRY;
3172
3173   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3174   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
3175
3176   if_addr = ip4_interface_first_address (ipm, sw_if_index, NULL);
3177
3178   if (NULL != if_addr)
3179     {
3180       fib_prefix_t pfx = {
3181         .fp_len = 32,
3182         .fp_proto = FIB_PROTOCOL_IP4,
3183         .fp_addr.ip4 = *if_addr,
3184       };
3185       u32 fib_index;
3186
3187       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3188                                                        sw_if_index);
3189
3190
3191       if (table_index != (u32) ~ 0)
3192         {
3193           dpo_id_t dpo = DPO_INVALID;
3194
3195           dpo_set (&dpo,
3196                    DPO_CLASSIFY,
3197                    DPO_PROTO_IP4,
3198                    classify_dpo_create (DPO_PROTO_IP4, table_index));
3199
3200           fib_table_entry_special_dpo_add (fib_index,
3201                                            &pfx,
3202                                            FIB_SOURCE_CLASSIFY,
3203                                            FIB_ENTRY_FLAG_NONE, &dpo);
3204           dpo_reset (&dpo);
3205         }
3206       else
3207         {
3208           fib_table_entry_special_remove (fib_index,
3209                                           &pfx, FIB_SOURCE_CLASSIFY);
3210         }
3211     }
3212
3213   return 0;
3214 }
3215
3216 static clib_error_t *
3217 set_ip_classify_command_fn (vlib_main_t * vm,
3218                             unformat_input_t * input,
3219                             vlib_cli_command_t * cmd)
3220 {
3221   u32 table_index = ~0;
3222   int table_index_set = 0;
3223   u32 sw_if_index = ~0;
3224   int rv;
3225
3226   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3227     {
3228       if (unformat (input, "table-index %d", &table_index))
3229         table_index_set = 1;
3230       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3231                          vnet_get_main (), &sw_if_index))
3232         ;
3233       else
3234         break;
3235     }
3236
3237   if (table_index_set == 0)
3238     return clib_error_return (0, "classify table-index must be specified");
3239
3240   if (sw_if_index == ~0)
3241     return clib_error_return (0, "interface / subif must be specified");
3242
3243   rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
3244
3245   switch (rv)
3246     {
3247     case 0:
3248       break;
3249
3250     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3251       return clib_error_return (0, "No such interface");
3252
3253     case VNET_API_ERROR_NO_SUCH_ENTRY:
3254       return clib_error_return (0, "No such classifier table");
3255     }
3256   return 0;
3257 }
3258
3259 /*?
3260  * Assign a classification table to an interface. The classification
3261  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3262  * commands. Once the table is create, use this command to filter packets
3263  * on an interface.
3264  *
3265  * @cliexpar
3266  * Example of how to assign a classification table to an interface:
3267  * @cliexcmd{set ip classify intfc GigabitEthernet2/0/0 table-index 1}
3268 ?*/
3269 /* *INDENT-OFF* */
3270 VLIB_CLI_COMMAND (set_ip_classify_command, static) =
3271 {
3272     .path = "set ip classify",
3273     .short_help =
3274     "set ip classify intfc <interface> table-index <classify-idx>",
3275     .function = set_ip_classify_command_fn,
3276 };
3277 /* *INDENT-ON* */
3278
3279 /*
3280  * fd.io coding-style-patch-verification: ON
3281  *
3282  * Local Variables:
3283  * eval: (c-set-style "gnu")
3284  * End:
3285  */