vnet/vnet/ip/ip4_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip4_forward.c: IP v4 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  43 #include <vnet/ethernet/arp_packet.h>   /* for ethernet_arp_header_t */
  44 #include <vnet/ppp/ppp.h>
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vnet/api_errno.h>     /* for API error numbers */
  47 #include <vnet/fib/fib_table.h> /* for FIB table and entry creation */
  48 #include <vnet/fib/fib_entry.h> /* for FIB table and entry creation */
  49 #include <vnet/fib/fib_urpf_list.h> /* for FIB uRPF check */
  50 #include <vnet/fib/ip4_fib.h>
  51 #include <vnet/dpo/load_balance.h>
  52 #include <vnet/dpo/classify_dpo.h>
  53
  54 /**
  55  * @file
  56  * @brief IPv4 Forwarding.
  57  *
  58  * This file contains the source code for IPv4 forwarding.
  59  */
  60
  61 void
  62 ip4_forward_next_trace (vlib_main_t * vm,
  63                         vlib_node_runtime_t * node,
  64                         vlib_frame_t * frame,
  65                         vlib_rx_or_tx_t which_adj_index);
  66
  67 always_inline uword
  68 ip4_lookup_inline (vlib_main_t * vm,
  69                    vlib_node_runtime_t * node,
  70                    vlib_frame_t * frame,
  71                    int lookup_for_responses_to_locally_received_packets)
  72 {
  73   ip4_main_t * im = &ip4_main;
  74   vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
  75   u32 n_left_from, n_left_to_next, * from, * to_next;
  76   ip_lookup_next_t next;
  77   u32 cpu_index = os_get_cpu_number();
  78
  79   from = vlib_frame_vector_args (frame);
  80   n_left_from = frame->n_vectors;
  81   next = node->cached_next_index;
  82
  83   while (n_left_from > 0)
  84     {
  85       vlib_get_next_frame (vm, node, next,
  86                            to_next, n_left_to_next);
  87
  88       while (n_left_from >= 8 && n_left_to_next >= 4)
  89         {
  90           vlib_buffer_t * p0, * p1, * p2, * p3;
  91           ip4_header_t * ip0, * ip1, * ip2, * ip3;
  92           __attribute__((unused)) tcp_header_t * tcp0, * tcp1, * tcp2, * tcp3;
  93           ip_lookup_next_t next0, next1, next2, next3;
  94           const load_balance_t * lb0, * lb1, * lb2, * lb3;
  95           ip4_fib_mtrie_t * mtrie0, * mtrie1, * mtrie2, * mtrie3;
  96           ip4_fib_mtrie_leaf_t leaf0, leaf1, leaf2, leaf3;
  97           ip4_address_t * dst_addr0, *dst_addr1, *dst_addr2, *dst_addr3;
  98           __attribute__((unused)) u32 pi0, fib_index0, lb_index0, is_tcp_udp0;
  99           __attribute__((unused)) u32 pi1, fib_index1, lb_index1, is_tcp_udp1;
 100           __attribute__((unused)) u32 pi2, fib_index2, lb_index2, is_tcp_udp2;
 101           __attribute__((unused)) u32 pi3, fib_index3, lb_index3, is_tcp_udp3;
 102           flow_hash_config_t flow_hash_config0, flow_hash_config1;
 103           flow_hash_config_t flow_hash_config2, flow_hash_config3;
 104           u32 hash_c0, hash_c1, hash_c2, hash_c3;
 105           const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3;
 106
 107           /* Prefetch next iteration. */
 108           {
 109             vlib_buffer_t * p4, * p5, * p6, * p7;
 110
 111             p4 = vlib_get_buffer (vm, from[4]);
 112             p5 = vlib_get_buffer (vm, from[5]);
 113             p6 = vlib_get_buffer (vm, from[6]);
 114             p7 = vlib_get_buffer (vm, from[7]);
 115
 116             vlib_prefetch_buffer_header (p4, LOAD);
 117             vlib_prefetch_buffer_header (p5, LOAD);
 118             vlib_prefetch_buffer_header (p6, LOAD);
 119             vlib_prefetch_buffer_header (p7, LOAD);
 120
 121             CLIB_PREFETCH (p4->data, sizeof (ip0[0]), LOAD);
 122             CLIB_PREFETCH (p5->data, sizeof (ip0[0]), LOAD);
 123             CLIB_PREFETCH (p6->data, sizeof (ip0[0]), LOAD);
 124             CLIB_PREFETCH (p7->data, sizeof (ip0[0]), LOAD);
 125           }
 126
 127           pi0 = to_next[0] = from[0];
 128           pi1 = to_next[1] = from[1];
 129           pi2 = to_next[2] = from[2];
 130           pi3 = to_next[3] = from[3];
 131
 132           from += 4;
 133           to_next += 4;
 134           n_left_to_next -= 4;
 135           n_left_from -= 4;
 136
 137           p0 = vlib_get_buffer (vm, pi0);
 138           p1 = vlib_get_buffer (vm, pi1);
 139           p2 = vlib_get_buffer (vm, pi2);
 140           p3 = vlib_get_buffer (vm, pi3);
 141
 142           ip0 = vlib_buffer_get_current (p0);
 143           ip1 = vlib_buffer_get_current (p1);
 144           ip2 = vlib_buffer_get_current (p2);
 145           ip3 = vlib_buffer_get_current (p3);
 146
 147           dst_addr0 = &ip0->dst_address;
 148           dst_addr1 = &ip1->dst_address;
 149           dst_addr2 = &ip2->dst_address;
 150           dst_addr3 = &ip3->dst_address;
 151
 152           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 153           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 154           fib_index2 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p2)->sw_if_index[VLIB_RX]);
 155           fib_index3 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p3)->sw_if_index[VLIB_RX]);
 156           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 157             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 158           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
 159             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
 160           fib_index2 = (vnet_buffer(p2)->sw_if_index[VLIB_TX] == (u32)~0) ?
 161             fib_index2 : vnet_buffer(p2)->sw_if_index[VLIB_TX];
 162           fib_index3 = (vnet_buffer(p3)->sw_if_index[VLIB_TX] == (u32)~0) ?
 163             fib_index3 : vnet_buffer(p3)->sw_if_index[VLIB_TX];
 164
 165
 166           if (! lookup_for_responses_to_locally_received_packets)
 167             {
 168               mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 169               mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
 170               mtrie2 = &ip4_fib_get (fib_index2)->mtrie;
 171               mtrie3 = &ip4_fib_get (fib_index3)->mtrie;
 172
 173               leaf0 = leaf1 = leaf2 = leaf3 = IP4_FIB_MTRIE_LEAF_ROOT;
 174
 175               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 0);
 176               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 0);
 177               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 0);
 178               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 0);
 179             }
 180
 181           tcp0 = (void *) (ip0 + 1);
 182           tcp1 = (void *) (ip1 + 1);
 183           tcp2 = (void *) (ip2 + 1);
 184           tcp3 = (void *) (ip3 + 1);
 185
 186           is_tcp_udp0 = (ip0->protocol == IP_PROTOCOL_TCP
 187                          || ip0->protocol == IP_PROTOCOL_UDP);
 188           is_tcp_udp1 = (ip1->protocol == IP_PROTOCOL_TCP
 189                          || ip1->protocol == IP_PROTOCOL_UDP);
 190           is_tcp_udp2 = (ip2->protocol == IP_PROTOCOL_TCP
 191                          || ip2->protocol == IP_PROTOCOL_UDP);
 192           is_tcp_udp3 = (ip1->protocol == IP_PROTOCOL_TCP
 193                          || ip1->protocol == IP_PROTOCOL_UDP);
 194
 195           if (! lookup_for_responses_to_locally_received_packets)
 196             {
 197               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
 198               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 1);
 199               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 1);
 200               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 1);
 201             }
 202
 203           if (! lookup_for_responses_to_locally_received_packets)
 204             {
 205               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 206               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
 207               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 2);
 208               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 2);
 209             }
 210
 211           if (! lookup_for_responses_to_locally_received_packets)
 212             {
 213               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 214               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
 215               leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 3);
 216               leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 3);
 217             }
 218
 219           if (lookup_for_responses_to_locally_received_packets)
 220             {
 221               lb_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 222               lb_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
 223               lb_index2 = vnet_buffer (p2)->ip.adj_index[VLIB_RX];
 224               lb_index3 = vnet_buffer (p3)->ip.adj_index[VLIB_RX];
 225             }
 226           else
 227             {
 228               /* Handle default route. */
 229               leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 230               leaf1 = (leaf1 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie1->default_leaf : leaf1);
 231               leaf2 = (leaf2 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie2->default_leaf : leaf2);
 232               leaf3 = (leaf3 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie3->default_leaf : leaf3);
 233               lb_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 234               lb_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 235               lb_index2 = ip4_fib_mtrie_leaf_get_adj_index (leaf2);
 236               lb_index3 = ip4_fib_mtrie_leaf_get_adj_index (leaf3);
 237             }
 238
 239           lb0 = load_balance_get (lb_index0);
 240           lb1 = load_balance_get (lb_index1);
 241           lb2 = load_balance_get (lb_index2);
 242           lb3 = load_balance_get (lb_index3);
 243
 244           /* Use flow hash to compute multipath adjacency. */
 245           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 246           hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
 247           hash_c2 = vnet_buffer (p2)->ip.flow_hash = 0;
 248           hash_c3 = vnet_buffer (p3)->ip.flow_hash = 0;
 249           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 250             {
 251               flow_hash_config0 = lb0->lb_hash_config;
 252               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 253                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 254             }
 255           if (PREDICT_FALSE(lb1->lb_n_buckets > 1))
 256             {
 257               flow_hash_config1 = lb1->lb_hash_config;
 258               hash_c1 = vnet_buffer (p1)->ip.flow_hash =
 259                 ip4_compute_flow_hash (ip1, flow_hash_config1);
 260             }
 261           if (PREDICT_FALSE (lb2->lb_n_buckets > 1))
 262             {
 263               flow_hash_config2 = lb2->lb_hash_config;
 264               hash_c2 = vnet_buffer (p2)->ip.flow_hash =
 265                 ip4_compute_flow_hash (ip2, flow_hash_config2);
 266             }
 267           if (PREDICT_FALSE(lb3->lb_n_buckets > 1))
 268             {
 269               flow_hash_config3 = lb3->lb_hash_config;
 270               hash_c3 = vnet_buffer (p3)->ip.flow_hash =
 271                 ip4_compute_flow_hash (ip3, flow_hash_config3);
 272             }
 273
 274           ASSERT (lb0->lb_n_buckets > 0);
 275           ASSERT (is_pow2 (lb0->lb_n_buckets));
 276           ASSERT (lb1->lb_n_buckets > 0);
 277           ASSERT (is_pow2 (lb1->lb_n_buckets));
 278           ASSERT (lb2->lb_n_buckets > 0);
 279           ASSERT (is_pow2 (lb2->lb_n_buckets));
 280           ASSERT (lb3->lb_n_buckets > 0);
 281           ASSERT (is_pow2 (lb3->lb_n_buckets));
 282
 283           dpo0 = load_balance_get_bucket_i(lb0,
 284                                            (hash_c0 &
 285                                             (lb0->lb_n_buckets_minus_1)));
 286           dpo1 = load_balance_get_bucket_i(lb1,
 287                                            (hash_c1 &
 288                                             (lb1->lb_n_buckets_minus_1)));
 289           dpo2 = load_balance_get_bucket_i(lb2,
 290                                            (hash_c2 &
 291                                             (lb2->lb_n_buckets_minus_1)));
 292           dpo3 = load_balance_get_bucket_i(lb3,
 293                                            (hash_c3 &
 294                                             (lb3->lb_n_buckets_minus_1)));
 295
 296           next0 = dpo0->dpoi_next_node;
 297           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 298           next1 = dpo1->dpoi_next_node;
 299           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 300           next2 = dpo2->dpoi_next_node;
 301           vnet_buffer (p2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
 302           next3 = dpo3->dpoi_next_node;
 303           vnet_buffer (p3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
 304
 305           vlib_increment_combined_counter
 306               (cm, cpu_index, lb_index0, 1,
 307                vlib_buffer_length_in_chain (vm, p0)
 308                + sizeof(ethernet_header_t));
 309           vlib_increment_combined_counter
 310               (cm, cpu_index, lb_index1, 1,
 311                vlib_buffer_length_in_chain (vm, p1)
 312                + sizeof(ethernet_header_t));
 313           vlib_increment_combined_counter
 314               (cm, cpu_index, lb_index2, 1,
 315                vlib_buffer_length_in_chain (vm, p2)
 316                + sizeof(ethernet_header_t));
 317           vlib_increment_combined_counter
 318               (cm, cpu_index, lb_index3, 1,
 319                vlib_buffer_length_in_chain (vm, p3)
 320                + sizeof(ethernet_header_t));
 321
 322           vlib_validate_buffer_enqueue_x4 (vm, node, next,
 323                                            to_next, n_left_to_next,
 324                                            pi0, pi1, pi2, pi3,
 325                                            next0, next1, next2, next3);
 326         }
 327
 328       while (n_left_from > 0 && n_left_to_next > 0)
 329         {
 330           vlib_buffer_t * p0;
 331           ip4_header_t * ip0;
 332           __attribute__((unused)) tcp_header_t * tcp0;
 333           ip_lookup_next_t next0;
 334           const load_balance_t *lb0;
 335           ip4_fib_mtrie_t * mtrie0;
 336           ip4_fib_mtrie_leaf_t leaf0;
 337           ip4_address_t * dst_addr0;
 338           __attribute__((unused)) u32 pi0, fib_index0, is_tcp_udp0, lbi0;
 339           flow_hash_config_t flow_hash_config0;
 340           const dpo_id_t *dpo0;
 341           u32 hash_c0;
 342
 343           pi0 = from[0];
 344           to_next[0] = pi0;
 345
 346           p0 = vlib_get_buffer (vm, pi0);
 347
 348           ip0 = vlib_buffer_get_current (p0);
 349
 350           dst_addr0 = &ip0->dst_address;
 351
 352           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 353           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 354             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 355
 356           if (! lookup_for_responses_to_locally_received_packets)
 357             {
 358               mtrie0 = &ip4_fib_get( fib_index0)->mtrie;
 359
 360               leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
 361
 362               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 0);
 363             }
 364
 365           tcp0 = (void *) (ip0 + 1);
 366
 367           is_tcp_udp0 = (ip0->protocol == IP_PROTOCOL_TCP
 368                          || ip0->protocol == IP_PROTOCOL_UDP);
 369
 370           if (! lookup_for_responses_to_locally_received_packets)
 371             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
 372
 373           if (! lookup_for_responses_to_locally_received_packets)
 374             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 375
 376           if (! lookup_for_responses_to_locally_received_packets)
 377             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 378
 379           if (lookup_for_responses_to_locally_received_packets)
 380             lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 381           else
 382             {
 383               /* Handle default route. */
 384               leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 385               lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 386             }
 387
 388           lb0 = load_balance_get (lbi0);
 389
 390           /* Use flow hash to compute multipath adjacency. */
 391           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 392           if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
 393             {
 394               flow_hash_config0 = lb0->lb_hash_config;
 395
 396               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 397                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 398             }
 399
 400           ASSERT (lb0->lb_n_buckets > 0);
 401           ASSERT (is_pow2 (lb0->lb_n_buckets));
 402
 403           dpo0 = load_balance_get_bucket_i(lb0,
 404                                            (hash_c0 &
 405                                             (lb0->lb_n_buckets_minus_1)));
 406
 407           next0 = dpo0->dpoi_next_node;
 408           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 409
 410           vlib_increment_combined_counter
 411               (cm, cpu_index, lbi0, 1,
 412                vlib_buffer_length_in_chain (vm, p0));
 413
 414           from += 1;
 415           to_next += 1;
 416           n_left_to_next -= 1;
 417           n_left_from -= 1;
 418
 419           if (PREDICT_FALSE (next0 != next))
 420             {
 421               n_left_to_next += 1;
 422               vlib_put_next_frame (vm, node, next, n_left_to_next);
 423               next = next0;
 424               vlib_get_next_frame (vm, node, next,
 425                                    to_next, n_left_to_next);
 426               to_next[0] = pi0;
 427               to_next += 1;
 428               n_left_to_next -= 1;
 429             }
 430         }
 431
 432       vlib_put_next_frame (vm, node, next, n_left_to_next);
 433     }
 434
 435   if (node->flags & VLIB_NODE_FLAG_TRACE)
 436     ip4_forward_next_trace(vm, node, frame, VLIB_TX);
 437
 438   return frame->n_vectors;
 439 }
 440
 441 /** @brief IPv4 lookup node.
 442     @node ip4-lookup
 443
 444     This is the main IPv4 lookup dispatch node.
 445
 446     @param vm vlib_main_t corresponding to the current thread
 447     @param node vlib_node_runtime_t
 448     @param frame vlib_frame_t whose contents should be dispatched
 449
 450     @par Graph mechanics: buffer metadata, next index usage
 451
 452     @em Uses:
 453     - <code>vnet_buffer(b)->sw_if_index[VLIB_RX]</code>
 454         - Indicates the @c sw_if_index value of the interface that the
 455           packet was received on.
 456     - <code>vnet_buffer(b)->sw_if_index[VLIB_TX]</code>
 457         - When the value is @c ~0 then the node performs a longest prefix
 458           match (LPM) for the packet destination address in the FIB attached
 459           to the receive interface.
 460         - Otherwise perform LPM for the packet destination address in the
 461           indicated FIB. In this case <code>[VLIB_TX]</code> is a FIB index
 462           value (0, 1, ...) and not a VRF id.
 463
 464     @em Sets:
 465     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
 466         - The lookup result adjacency index.
 467
 468     <em>Next Index:</em>
 469     - Dispatches the packet to the node index found in
 470       ip_adjacency_t @c adj->lookup_next_index
 471       (where @c adj is the lookup result adjacency).
 472 */
 473 static uword
 474 ip4_lookup (vlib_main_t * vm,
 475             vlib_node_runtime_t * node,
 476             vlib_frame_t * frame)
 477 {
 478   return ip4_lookup_inline (vm, node, frame,
 479                             /* lookup_for_responses_to_locally_received_packets */ 0);
 480
 481 }
 482
 483 static u8 * format_ip4_lookup_trace (u8 * s, va_list * args);
 484
 485 VLIB_REGISTER_NODE (ip4_lookup_node) = {
 486   .function = ip4_lookup,
 487   .name = "ip4-lookup",
 488   .vector_size = sizeof (u32),
 489
 490   .format_trace = format_ip4_lookup_trace,
 491   .n_next_nodes = IP_LOOKUP_N_NEXT,
 492   .next_nodes = IP4_LOOKUP_NEXT_NODES,
 493 };
 494
 495 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup)
 496
 497 always_inline uword
 498 ip4_load_balance (vlib_main_t * vm,
 499                   vlib_node_runtime_t * node,
 500                   vlib_frame_t * frame)
 501 {
 502   vlib_combined_counter_main_t * cm = &load_balance_main.lbm_via_counters;
 503   u32 n_left_from, n_left_to_next, * from, * to_next;
 504   ip_lookup_next_t next;
 505   u32 cpu_index = os_get_cpu_number();
 506
 507   from = vlib_frame_vector_args (frame);
 508   n_left_from = frame->n_vectors;
 509   next = node->cached_next_index;
 510
 511   if (node->flags & VLIB_NODE_FLAG_TRACE)
 512       ip4_forward_next_trace(vm, node, frame, VLIB_TX);
 513
 514   while (n_left_from > 0)
 515     {
 516       vlib_get_next_frame (vm, node, next,
 517                            to_next, n_left_to_next);
 518
 519
 520       while (n_left_from >= 4 && n_left_to_next >= 2)
 521         {
 522           ip_lookup_next_t next0, next1;
 523           const load_balance_t *lb0, *lb1;
 524           vlib_buffer_t * p0, *p1;
 525           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 526           const ip4_header_t *ip0, *ip1;
 527           const dpo_id_t *dpo0, *dpo1;
 528
 529           /* Prefetch next iteration. */
 530           {
 531             vlib_buffer_t * p2, * p3;
 532
 533             p2 = vlib_get_buffer (vm, from[2]);
 534             p3 = vlib_get_buffer (vm, from[3]);
 535
 536             vlib_prefetch_buffer_header (p2, STORE);
 537             vlib_prefetch_buffer_header (p3, STORE);
 538
 539             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 540             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 541           }
 542
 543           pi0 = to_next[0] = from[0];
 544           pi1 = to_next[1] = from[1];
 545
 546           from += 2;
 547           n_left_from -= 2;
 548           to_next += 2;
 549           n_left_to_next -= 2;
 550
 551           p0 = vlib_get_buffer (vm, pi0);
 552           p1 = vlib_get_buffer (vm, pi1);
 553
 554           ip0 = vlib_buffer_get_current (p0);
 555           ip1 = vlib_buffer_get_current (p1);
 556           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 557           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 558
 559           lb0 = load_balance_get(lbi0);
 560           lb1 = load_balance_get(lbi1);
 561
 562           /*
 563            * this node is for via FIBs we can re-use the hash value from the
 564            * to node if present.
 565            * We don't want to use the same hash value at each level in the recursion
 566            * graph as that would lead to polarisation
 567            */
 568           hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
 569           hc1 = vnet_buffer (p1)->ip.flow_hash = 0;
 570
 571           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 572           {
 573               if (PREDICT_TRUE (vnet_buffer(p0)->ip.flow_hash))
 574               {
 575                   hc0 = vnet_buffer(p0)->ip.flow_hash = vnet_buffer(p0)->ip.flow_hash >> 1;
 576               }
 577               else
 578               {
 579                   hc0 = vnet_buffer(p0)->ip.flow_hash = ip4_compute_flow_hash(ip0, hc0);
 580               }
 581           }
 582           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 583           {
 584               if (PREDICT_TRUE (vnet_buffer(p1)->ip.flow_hash))
 585               {
 586                   hc1 = vnet_buffer(p1)->ip.flow_hash = vnet_buffer(p1)->ip.flow_hash >> 1;
 587               }
 588               else
 589               {
 590                   hc1 = vnet_buffer(p1)->ip.flow_hash = ip4_compute_flow_hash(ip1, hc1);
 591               }
 592           }
 593
 594           dpo0 = load_balance_get_bucket_i(lb0, hc0 & (lb0->lb_n_buckets_minus_1));
 595           dpo1 = load_balance_get_bucket_i(lb1, hc1 & (lb1->lb_n_buckets_minus_1));
 596
 597           next0 = dpo0->dpoi_next_node;
 598           next1 = dpo1->dpoi_next_node;
 599
 600           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 601           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 602
 603           vlib_increment_combined_counter
 604               (cm, cpu_index, lbi0, 1,
 605                vlib_buffer_length_in_chain (vm, p0));
 606           vlib_increment_combined_counter
 607               (cm, cpu_index, lbi1, 1,
 608                vlib_buffer_length_in_chain (vm, p1));
 609
 610           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 611                                            to_next, n_left_to_next,
 612                                            pi0, pi1, next0, next1);
 613        }
 614
 615       while (n_left_from > 0 && n_left_to_next > 0)
 616         {
 617           ip_lookup_next_t next0;
 618           const load_balance_t *lb0;
 619           vlib_buffer_t * p0;
 620           u32 pi0, lbi0, hc0;
 621           const ip4_header_t *ip0;
 622           const dpo_id_t *dpo0;
 623
 624           pi0 = from[0];
 625           to_next[0] = pi0;
 626           from += 1;
 627           to_next += 1;
 628           n_left_to_next -= 1;
 629           n_left_from -= 1;
 630
 631           p0 = vlib_get_buffer (vm, pi0);
 632
 633           ip0 = vlib_buffer_get_current (p0);
 634           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 635
 636           lb0 = load_balance_get(lbi0);
 637
 638           hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
 639           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 640           {
 641               if (PREDICT_TRUE (vnet_buffer(p0)->ip.flow_hash))
 642               {
 643                   hc0 = vnet_buffer(p0)->ip.flow_hash = vnet_buffer(p0)->ip.flow_hash >> 1;
 644               }
 645               else
 646               {
 647                   hc0 = vnet_buffer(p0)->ip.flow_hash = ip4_compute_flow_hash(ip0, hc0);
 648               }
 649           }
 650
 651           dpo0 = load_balance_get_bucket_i(lb0, hc0 & (lb0->lb_n_buckets_minus_1));
 652
 653           next0 = dpo0->dpoi_next_node;
 654           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 655
 656           vlib_increment_combined_counter
 657               (cm, cpu_index, lbi0, 1,
 658                vlib_buffer_length_in_chain (vm, p0));
 659
 660           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 661                                            to_next, n_left_to_next,
 662                                            pi0, next0);
 663         }
 664
 665       vlib_put_next_frame (vm, node, next, n_left_to_next);
 666     }
 667
 668   return frame->n_vectors;
 669 }
 670
 671 VLIB_REGISTER_NODE (ip4_load_balance_node) = {
 672   .function = ip4_load_balance,
 673   .name = "ip4-load-balance",
 674   .vector_size = sizeof (u32),
 675   .sibling_of = "ip4-lookup",
 676
 677   .format_trace = format_ip4_lookup_trace,
 678 };
 679
 680 VLIB_NODE_FUNCTION_MULTIARCH (ip4_load_balance_node, ip4_load_balance)
 681
 682 /* get first interface address */
 683 ip4_address_t *
 684 ip4_interface_first_address (ip4_main_t * im, u32 sw_if_index,
 685                              ip_interface_address_t ** result_ia)
 686 {
 687   ip_lookup_main_t * lm = &im->lookup_main;
 688   ip_interface_address_t * ia = 0;
 689   ip4_address_t * result = 0;
 690
 691   foreach_ip_interface_address (lm, ia, sw_if_index,
 692                                 1 /* honor unnumbered */,
 693   ({
 694     ip4_address_t * a = ip_interface_address_get_address (lm, ia);
 695     result = a;
 696     break;
 697   }));
 698   if (result_ia)
 699     *result_ia = result ? ia : 0;
 700   return result;
 701 }
 702
 703 static void
 704 ip4_add_interface_routes (u32 sw_if_index,
 705                           ip4_main_t * im, u32 fib_index,
 706                           ip_interface_address_t * a)
 707 {
 708   ip_lookup_main_t * lm = &im->lookup_main;
 709   ip4_address_t * address = ip_interface_address_get_address (lm, a);
 710   fib_prefix_t pfx = {
 711       .fp_len = a->address_length,
 712       .fp_proto = FIB_PROTOCOL_IP4,
 713       .fp_addr.ip4 = *address,
 714   };
 715
 716   a->neighbor_probe_adj_index = ~0;
 717
 718   if (pfx.fp_len < 32)
 719   {
 720       fib_node_index_t fei;
 721
 722       fei = fib_table_entry_update_one_path(fib_index,
 723                                             &pfx,
 724                                             FIB_SOURCE_INTERFACE,
 725                                             (FIB_ENTRY_FLAG_CONNECTED |
 726                                              FIB_ENTRY_FLAG_ATTACHED),
 727                                             FIB_PROTOCOL_IP4,
 728                                             NULL, /* No next-hop address */
 729                                             sw_if_index,
 730                                             ~0, // invalid FIB index
 731                                             1,
 732                                             MPLS_LABEL_INVALID,
 733                                             FIB_ROUTE_PATH_FLAG_NONE);
 734       a->neighbor_probe_adj_index = fib_entry_get_adj(fei);
 735   }
 736
 737   pfx.fp_len = 32;
 738
 739   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 740   {
 741       u32 classify_table_index =
 742           lm->classify_table_index_by_sw_if_index [sw_if_index];
 743       if (classify_table_index != (u32) ~0)
 744       {
 745           dpo_id_t dpo = DPO_INVALID;
 746
 747           dpo_set(&dpo,
 748                   DPO_CLASSIFY,
 749                   DPO_PROTO_IP4,
 750                   classify_dpo_create(DPO_PROTO_IP4,
 751                                       classify_table_index));
 752
 753           fib_table_entry_special_dpo_add(fib_index,
 754                                           &pfx,
 755                                           FIB_SOURCE_CLASSIFY,
 756                                           FIB_ENTRY_FLAG_NONE,
 757                                           &dpo);
 758           dpo_reset(&dpo);
 759       }
 760   }
 761
 762   fib_table_entry_update_one_path(fib_index,
 763                                   &pfx,
 764                                   FIB_SOURCE_INTERFACE,
 765                                   (FIB_ENTRY_FLAG_CONNECTED |
 766                                    FIB_ENTRY_FLAG_LOCAL),
 767                                   FIB_PROTOCOL_IP4,
 768                                   &pfx.fp_addr,
 769                                   sw_if_index,
 770                                   ~0, // invalid FIB index
 771                                   1,
 772                                   MPLS_LABEL_INVALID,
 773                                   FIB_ROUTE_PATH_FLAG_NONE);
 774 }
 775
 776 static void
 777 ip4_del_interface_routes (ip4_main_t * im,
 778                           u32 fib_index,
 779                           ip4_address_t * address,
 780                           u32 address_length)
 781 {
 782     fib_prefix_t pfx = {
 783         .fp_len = address_length,
 784         .fp_proto = FIB_PROTOCOL_IP4,
 785         .fp_addr.ip4 = *address,
 786     };
 787
 788     if (pfx.fp_len < 32)
 789     {
 790         fib_table_entry_delete(fib_index,
 791                                &pfx,
 792                                FIB_SOURCE_INTERFACE);
 793     }
 794
 795     pfx.fp_len = 32;
 796     fib_table_entry_delete(fib_index,
 797                            &pfx,
 798                            FIB_SOURCE_INTERFACE);
 799 }
 800
 801 void
 802 ip4_sw_interface_enable_disable (u32 sw_if_index,
 803                                  u32 is_enable)
 804 {
 805   ip4_main_t * im = &ip4_main;
 806
 807   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 808
 809   /*
 810    * enable/disable only on the 1<->0 transition
 811    */
 812   if (is_enable)
 813     {
 814       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 815         return;
 816     }
 817   else
 818     {
 819       ASSERT(im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 820       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 821         return;
 822     }
 823   vnet_feature_enable_disable ("ip4-unicast", "ip4-lookup", sw_if_index,
 824                                is_enable, 0, 0);
 825
 826   vnet_feature_enable_disable ("ip4-multicast", "ip4-lookup-multicast", sw_if_index,
 827                                is_enable, 0, 0);
 828
 829 }
 830
 831 static clib_error_t *
 832 ip4_add_del_interface_address_internal (vlib_main_t * vm,
 833                                         u32 sw_if_index,
 834                                         ip4_address_t * address,
 835                                         u32 address_length,
 836                                         u32 is_del)
 837 {
 838   vnet_main_t * vnm = vnet_get_main();
 839   ip4_main_t * im = &ip4_main;
 840   ip_lookup_main_t * lm = &im->lookup_main;
 841   clib_error_t * error = 0;
 842   u32 if_address_index, elts_before;
 843   ip4_address_fib_t ip4_af, * addr_fib = 0;
 844
 845   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 846   ip4_addr_fib_init (&ip4_af, address,
 847                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 848   vec_add1 (addr_fib, ip4_af);
 849
 850   /* FIXME-LATER
 851    * there is no support for adj-fib handling in the presence of overlapping
 852    * subnets on interfaces. Easy fix - disallow overlapping subnets, like
 853    * most routers do.
 854    */
 855   if (! is_del)
 856     {
 857       /* When adding an address check that it does not conflict
 858          with an existing address. */
 859       ip_interface_address_t * ia;
 860       foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
 861                                     0 /* honor unnumbered */,
 862       ({
 863         ip4_address_t * x = ip_interface_address_get_address (&im->lookup_main, ia);
 864
 865         if (ip4_destination_matches_route (im, address, x, ia->address_length)
 866             || ip4_destination_matches_route (im, x, address, address_length))
 867           return clib_error_create ("failed to add %U which conflicts with %U for interface %U",
 868                                     format_ip4_address_and_length, address, address_length,
 869                                     format_ip4_address_and_length, x, ia->address_length,
 870                                     format_vnet_sw_if_index_name, vnm, sw_if_index);
 871        }));
 872     }
 873
 874   elts_before = pool_elts (lm->if_address_pool);
 875
 876   error = ip_interface_address_add_del
 877     (lm,
 878      sw_if_index,
 879      addr_fib,
 880      address_length,
 881      is_del,
 882      &if_address_index);
 883   if (error)
 884     goto done;
 885
 886   ip4_sw_interface_enable_disable(sw_if_index, !is_del);
 887
 888   if (is_del)
 889       ip4_del_interface_routes (im, ip4_af.fib_index, address,
 890                                 address_length);
 891   else
 892       ip4_add_interface_routes (sw_if_index,
 893                                 im, ip4_af.fib_index,
 894                                 pool_elt_at_index
 895                                 (lm->if_address_pool, if_address_index));
 896
 897   /* If pool did not grow/shrink: add duplicate address. */
 898   if (elts_before != pool_elts (lm->if_address_pool))
 899     {
 900       ip4_add_del_interface_address_callback_t * cb;
 901       vec_foreach (cb, im->add_del_interface_address_callbacks)
 902         cb->function (im, cb->function_opaque, sw_if_index,
 903                       address, address_length,
 904                       if_address_index,
 905                       is_del);
 906     }
 907
 908  done:
 909   vec_free (addr_fib);
 910   return error;
 911 }
 912
 913 clib_error_t *
 914 ip4_add_del_interface_address (vlib_main_t * vm, u32 sw_if_index,
 915                                ip4_address_t * address, u32 address_length,
 916                                u32 is_del)
 917 {
 918   return ip4_add_del_interface_address_internal
 919     (vm, sw_if_index, address, address_length,
 920      is_del);
 921 }
 922
 923 /* Built-in ip4 unicast rx feature path definition */
 924 VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
 925 {
 926   .arc_name  = "ip4-unicast",
 927   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 928   .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
 929 };
 930
 931 VNET_FEATURE_INIT (ip4_flow_classify, static) = {
 932   .arc_name = "ip4-unicast",
 933   .node_name = "ip4-flow-classify",
 934   .runs_before = VNET_FEATURES ("ip4-inacl"),
 935 };
 936
 937 VNET_FEATURE_INIT (ip4_inacl, static) = {
 938   .arc_name = "ip4-unicast",
 939   .node_name = "ip4-inacl",
 940   .runs_before = VNET_FEATURES ("ip4-source-check-via-rx"),
 941 };
 942
 943 VNET_FEATURE_INIT (ip4_source_check_1, static) = {
 944   .arc_name = "ip4-unicast",
 945   .node_name = "ip4-source-check-via-rx",
 946   .runs_before = VNET_FEATURES ("ip4-source-check-via-any"),
 947 };
 948
 949 VNET_FEATURE_INIT (ip4_source_check_2, static) = {
 950   .arc_name = "ip4-unicast",
 951   .node_name = "ip4-source-check-via-any",
 952   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
 953 };
 954
 955 VNET_FEATURE_INIT (ip4_source_and_port_range_check_rx, static) = {
 956   .arc_name = "ip4-unicast",
 957   .node_name = "ip4-source-and-port-range-check-rx",
 958   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
 959 };
 960
 961 VNET_FEATURE_INIT (ip4_policer_classify, static) = {
 962   .arc_name = "ip4-unicast",
 963   .node_name = "ip4-policer-classify",
 964   .runs_before = VNET_FEATURES ("ipsec-input-ip4"),
 965 };
 966
 967 VNET_FEATURE_INIT (ip4_ipsec, static) = {
 968   .arc_name = "ip4-unicast",
 969   .node_name = "ipsec-input-ip4",
 970   .runs_before = VNET_FEATURES ("vpath-input-ip4"),
 971 };
 972
 973 VNET_FEATURE_INIT (ip4_vpath, static) = {
 974   .arc_name = "ip4-unicast",
 975   .node_name = "vpath-input-ip4",
 976   .runs_before = VNET_FEATURES ("ip4-lookup"),
 977 };
 978
 979 VNET_FEATURE_INIT (ip4_lookup, static) = {
 980   .arc_name = "ip4-unicast",
 981   .node_name = "ip4-lookup",
 982   .runs_before = VNET_FEATURES ("ip4-drop"),
 983 };
 984
 985 VNET_FEATURE_INIT (ip4_drop, static) = {
 986   .arc_name = "ip4-unicast",
 987   .node_name = "ip4-drop",
 988   .runs_before = 0, /* not before any other features */
 989 };
 990
 991
 992 /* Built-in ip4 multicast rx feature path definition */
 993 VNET_FEATURE_ARC_INIT (ip4_multicast, static) =
 994 {
 995   .arc_name  = "ip4-multicast",
 996   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 997   .arc_index_ptr = &ip4_main.lookup_main.mcast_feature_arc_index,
 998 };
 999
1000 VNET_FEATURE_INIT (ip4_vpath_mc, static) = {
1001   .arc_name = "ip4-multicast",
1002   .node_name = "vpath-input-ip4",
1003   .runs_before = VNET_FEATURES ("ip4-lookup-multicast"),
1004 };
1005
1006 VNET_FEATURE_INIT (ip4_lookup_mc, static) = {
1007   .arc_name = "ip4-multicast",
1008   .node_name = "ip4-lookup-multicast",
1009   .runs_before = VNET_FEATURES ("ip4-drop"),
1010 };
1011
1012 VNET_FEATURE_INIT (ip4_mc_drop, static) = {
1013   .arc_name = "ip4-multicast",
1014   .node_name = "ip4-drop",
1015   .runs_before = 0, /* last feature */
1016 };
1017
1018 /* Source and port-range check ip4 tx feature path definition */
1019 VNET_FEATURE_ARC_INIT (ip4_output, static) =
1020 {
1021   .arc_name  = "ip4-output",
1022   .start_nodes = VNET_FEATURES ("ip4-rewrite-transit", "ip4-midchain"),
1023   .arc_index_ptr = &ip4_main.lookup_main.output_feature_arc_index,
1024 };
1025
1026 VNET_FEATURE_INIT (ip4_source_and_port_range_check_tx, static) = {
1027   .arc_name = "ip4-output",
1028   .node_name = "ip4-source-and-port-range-check-tx",
1029   .runs_before = VNET_FEATURES ("ipsec-output-ip4"),
1030 };
1031
1032 VNET_FEATURE_INIT (ip4_ipsec_output, static) = {
1033   .arc_name = "ip4-output",
1034   .node_name = "ipsec-output-ip4",
1035   .runs_before = VNET_FEATURES ("interface-output"),
1036 };
1037
1038 /* Built-in ip4 tx feature path definition */
1039 VNET_FEATURE_INIT (ip4_interface_output, static) = {
1040   .arc_name = "ip4-output",
1041   .node_name = "interface-output",
1042   .runs_before = 0, /* not before any other features */
1043 };
1044
1045
1046 static clib_error_t *
1047 ip4_sw_interface_add_del (vnet_main_t * vnm,
1048                           u32 sw_if_index,
1049                           u32 is_add)
1050 {
1051   ip4_main_t * im = &ip4_main;
1052
1053   /* Fill in lookup tables with default table (0). */
1054   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1055
1056   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
1057                                is_add, 0, 0);
1058
1059   vnet_feature_enable_disable ("ip4-multicast", "ip4-drop", sw_if_index,
1060                                is_add, 0, 0);
1061
1062   vnet_feature_enable_disable ("ip4-output", "interface-output", sw_if_index,
1063                                is_add, 0, 0);
1064
1065   return /* no error */ 0;
1066 }
1067
1068 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
1069
1070 /* Global IP4 main. */
1071 ip4_main_t ip4_main;
1072
1073 clib_error_t *
1074 ip4_lookup_init (vlib_main_t * vm)
1075 {
1076   ip4_main_t * im = &ip4_main;
1077   clib_error_t * error;
1078   uword i;
1079
1080   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
1081     return error;
1082
1083   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
1084     {
1085       u32 m;
1086
1087       if (i < 32)
1088         m = pow2_mask (i) << (32 - i);
1089       else
1090         m = ~0;
1091       im->fib_masks[i] = clib_host_to_net_u32 (m);
1092     }
1093
1094   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
1095
1096   /* Create FIB with index 0 and table id of 0. */
1097   fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP4, 0);
1098
1099   {
1100     pg_node_t * pn;
1101     pn = pg_get_node (ip4_lookup_node.index);
1102     pn->unformat_edit = unformat_pg_ip4_header;
1103   }
1104
1105   {
1106     ethernet_arp_header_t h;
1107
1108     memset (&h, 0, sizeof (h));
1109
1110     /* Set target ethernet address to all zeros. */
1111     memset (h.ip4_over_ethernet[1].ethernet, 0, sizeof (h.ip4_over_ethernet[1].ethernet));
1112
1113 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
1114 #define _8(f,v) h.f = v;
1115     _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
1116     _16 (l3_type, ETHERNET_TYPE_IP4);
1117     _8 (n_l2_address_bytes, 6);
1118     _8 (n_l3_address_bytes, 4);
1119     _16 (opcode, ETHERNET_ARP_OPCODE_request);
1120 #undef _16
1121 #undef _8
1122
1123     vlib_packet_template_init (vm,
1124                                &im->ip4_arp_request_packet_template,
1125                                /* data */ &h,
1126                                sizeof (h),
1127                                /* alloc chunk size */ 8,
1128                                "ip4 arp");
1129   }
1130
1131   return error;
1132 }
1133
1134 VLIB_INIT_FUNCTION (ip4_lookup_init);
1135
1136 typedef struct {
1137   /* Adjacency taken. */
1138   u32 dpo_index;
1139   u32 flow_hash;
1140   u32 fib_index;
1141
1142   /* Packet data, possibly *after* rewrite. */
1143   u8 packet_data[64 - 1*sizeof(u32)];
1144 } ip4_forward_next_trace_t;
1145
1146 static u8 * format_ip4_forward_next_trace (u8 * s, va_list * args)
1147 {
1148   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1149   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1150   ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1151   uword indent = format_get_indent (s);
1152   s = format (s, "%U%U",
1153               format_white_space, indent,
1154               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1155   return s;
1156 }
1157
1158 static u8 * format_ip4_lookup_trace (u8 * s, va_list * args)
1159 {
1160   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1161   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1162   ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1163   uword indent = format_get_indent (s);
1164
1165   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
1166               t->fib_index, t->dpo_index, t->flow_hash);
1167   s = format (s, "\n%U%U",
1168               format_white_space, indent,
1169               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1170   return s;
1171 }
1172
1173 static u8 * format_ip4_rewrite_trace (u8 * s, va_list * args)
1174 {
1175   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1176   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1177   ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1178   vnet_main_t * vnm = vnet_get_main();
1179   uword indent = format_get_indent (s);
1180
1181   s = format (s, "tx_sw_if_index %d dpo-idx %d : %U flow hash: 0x%08x",
1182               t->fib_index, t->dpo_index, format_ip_adjacency,
1183               t->dpo_index, FORMAT_IP_ADJACENCY_NONE,
1184               t->flow_hash);
1185   s = format (s, "\n%U%U",
1186               format_white_space, indent,
1187               format_ip_adjacency_packet_data,
1188               vnm, t->dpo_index,
1189               t->packet_data, sizeof (t->packet_data));
1190   return s;
1191 }
1192
1193 /* Common trace function for all ip4-forward next nodes. */
1194 void
1195 ip4_forward_next_trace (vlib_main_t * vm,
1196                         vlib_node_runtime_t * node,
1197                         vlib_frame_t * frame,
1198                         vlib_rx_or_tx_t which_adj_index)
1199 {
1200   u32 * from, n_left;
1201   ip4_main_t * im = &ip4_main;
1202
1203   n_left = frame->n_vectors;
1204   from = vlib_frame_vector_args (frame);
1205
1206   while (n_left >= 4)
1207     {
1208       u32 bi0, bi1;
1209       vlib_buffer_t * b0, * b1;
1210       ip4_forward_next_trace_t * t0, * t1;
1211
1212       /* Prefetch next iteration. */
1213       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1214       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1215
1216       bi0 = from[0];
1217       bi1 = from[1];
1218
1219       b0 = vlib_get_buffer (vm, bi0);
1220       b1 = vlib_get_buffer (vm, bi1);
1221
1222       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1223         {
1224           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1225           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1226           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1227           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1228               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1229               vec_elt (im->fib_index_by_sw_if_index,
1230                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1231
1232           clib_memcpy (t0->packet_data,
1233                   vlib_buffer_get_current (b0),
1234                   sizeof (t0->packet_data));
1235         }
1236       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1237         {
1238           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1239           t1->dpo_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1240           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1241           t1->fib_index = (vnet_buffer(b1)->sw_if_index[VLIB_TX] != (u32)~0) ?
1242               vnet_buffer(b1)->sw_if_index[VLIB_TX] :
1243               vec_elt (im->fib_index_by_sw_if_index,
1244                        vnet_buffer(b1)->sw_if_index[VLIB_RX]);
1245           clib_memcpy (t1->packet_data,
1246                   vlib_buffer_get_current (b1),
1247                   sizeof (t1->packet_data));
1248         }
1249       from += 2;
1250       n_left -= 2;
1251     }
1252
1253   while (n_left >= 1)
1254     {
1255       u32 bi0;
1256       vlib_buffer_t * b0;
1257       ip4_forward_next_trace_t * t0;
1258
1259       bi0 = from[0];
1260
1261       b0 = vlib_get_buffer (vm, bi0);
1262
1263       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1264         {
1265           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1266           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1267           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1268           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1269               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1270               vec_elt (im->fib_index_by_sw_if_index,
1271                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1272           clib_memcpy (t0->packet_data,
1273                   vlib_buffer_get_current (b0),
1274                   sizeof (t0->packet_data));
1275         }
1276       from += 1;
1277       n_left -= 1;
1278     }
1279 }
1280
1281 static uword
1282 ip4_drop_or_punt (vlib_main_t * vm,
1283                   vlib_node_runtime_t * node,
1284                   vlib_frame_t * frame,
1285                   ip4_error_t error_code)
1286 {
1287   u32 * buffers = vlib_frame_vector_args (frame);
1288   uword n_packets = frame->n_vectors;
1289
1290   vlib_error_drop_buffers (vm, node,
1291                            buffers,
1292                            /* stride */ 1,
1293                            n_packets,
1294                            /* next */ 0,
1295                            ip4_input_node.index,
1296                            error_code);
1297
1298   if (node->flags & VLIB_NODE_FLAG_TRACE)
1299     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1300
1301   return n_packets;
1302 }
1303
1304 static uword
1305 ip4_drop (vlib_main_t * vm,
1306           vlib_node_runtime_t * node,
1307           vlib_frame_t * frame)
1308 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_DROP); }
1309
1310 static uword
1311 ip4_punt (vlib_main_t * vm,
1312           vlib_node_runtime_t * node,
1313           vlib_frame_t * frame)
1314 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_PUNT); }
1315
1316 VLIB_REGISTER_NODE (ip4_drop_node,static) = {
1317   .function = ip4_drop,
1318   .name = "ip4-drop",
1319   .vector_size = sizeof (u32),
1320
1321   .format_trace = format_ip4_forward_next_trace,
1322
1323   .n_next_nodes = 1,
1324   .next_nodes = {
1325     [0] = "error-drop",
1326   },
1327 };
1328
1329 VLIB_NODE_FUNCTION_MULTIARCH (ip4_drop_node, ip4_drop)
1330
1331 VLIB_REGISTER_NODE (ip4_punt_node,static) = {
1332   .function = ip4_punt,
1333   .name = "ip4-punt",
1334   .vector_size = sizeof (u32),
1335
1336   .format_trace = format_ip4_forward_next_trace,
1337
1338   .n_next_nodes = 1,
1339   .next_nodes = {
1340     [0] = "error-punt",
1341   },
1342 };
1343
1344 VLIB_NODE_FUNCTION_MULTIARCH (ip4_punt_node, ip4_punt)
1345
1346 /* Compute TCP/UDP/ICMP4 checksum in software. */
1347 u16
1348 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1349                               ip4_header_t * ip0)
1350 {
1351   ip_csum_t sum0;
1352   u32 ip_header_length, payload_length_host_byte_order;
1353   u32 n_this_buffer, n_bytes_left;
1354   u16 sum16;
1355   void * data_this_buffer;
1356
1357   /* Initialize checksum with ip header. */
1358   ip_header_length = ip4_header_bytes (ip0);
1359   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->length) - ip_header_length;
1360   sum0 = clib_host_to_net_u32 (payload_length_host_byte_order + (ip0->protocol << 16));
1361
1362   if (BITS (uword) == 32)
1363     {
1364       sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u32));
1365       sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->dst_address, u32));
1366     }
1367   else
1368     sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1369
1370   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1371   data_this_buffer = (void *) ip0 + ip_header_length;
1372   if (n_this_buffer + ip_header_length > p0->current_length)
1373     n_this_buffer = p0->current_length > ip_header_length ? p0->current_length - ip_header_length : 0;
1374   while (1)
1375     {
1376       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1377       n_bytes_left -= n_this_buffer;
1378       if (n_bytes_left == 0)
1379         break;
1380
1381       ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1382       p0 = vlib_get_buffer (vm, p0->next_buffer);
1383       data_this_buffer = vlib_buffer_get_current (p0);
1384       n_this_buffer = p0->current_length;
1385     }
1386
1387   sum16 = ~ ip_csum_fold (sum0);
1388
1389   return sum16;
1390 }
1391
1392 static u32
1393 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1394 {
1395   ip4_header_t * ip0 = vlib_buffer_get_current (p0);
1396   udp_header_t * udp0;
1397   u16 sum16;
1398
1399   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1400           || ip0->protocol == IP_PROTOCOL_UDP);
1401
1402   udp0 = (void *) (ip0 + 1);
1403   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1404     {
1405       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1406                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1407       return p0->flags;
1408     }
1409
1410   sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1411
1412   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1413                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1414
1415   return p0->flags;
1416 }
1417
1418 static uword
1419 ip4_local (vlib_main_t * vm,
1420            vlib_node_runtime_t * node,
1421            vlib_frame_t * frame)
1422 {
1423   ip4_main_t * im = &ip4_main;
1424   ip_lookup_main_t * lm = &im->lookup_main;
1425   ip_local_next_t next_index;
1426   u32 * from, * to_next, n_left_from, n_left_to_next;
1427   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip4_input_node.index);
1428
1429   from = vlib_frame_vector_args (frame);
1430   n_left_from = frame->n_vectors;
1431   next_index = node->cached_next_index;
1432
1433   if (node->flags & VLIB_NODE_FLAG_TRACE)
1434     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1435
1436   while (n_left_from > 0)
1437     {
1438       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1439
1440       while (n_left_from >= 4 && n_left_to_next >= 2)
1441         {
1442           vlib_buffer_t * p0, * p1;
1443           ip4_header_t * ip0, * ip1;
1444           udp_header_t * udp0, * udp1;
1445           ip4_fib_mtrie_t * mtrie0, * mtrie1;
1446           ip4_fib_mtrie_leaf_t leaf0, leaf1;
1447           const dpo_id_t *dpo0, *dpo1;
1448           const load_balance_t *lb0, *lb1;
1449           u32 pi0, ip_len0, udp_len0, flags0, next0, fib_index0, lbi0;
1450           u32 pi1, ip_len1, udp_len1, flags1, next1, fib_index1, lbi1;
1451           i32 len_diff0, len_diff1;
1452           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1453           u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1454           u8 enqueue_code;
1455
1456           pi0 = to_next[0] = from[0];
1457           pi1 = to_next[1] = from[1];
1458           from += 2;
1459           n_left_from -= 2;
1460           to_next += 2;
1461           n_left_to_next -= 2;
1462
1463           p0 = vlib_get_buffer (vm, pi0);
1464           p1 = vlib_get_buffer (vm, pi1);
1465
1466           ip0 = vlib_buffer_get_current (p0);
1467           ip1 = vlib_buffer_get_current (p1);
1468
1469           fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
1470                                 vnet_buffer(p0)->sw_if_index[VLIB_RX]);
1471           fib_index1 = vec_elt (im->fib_index_by_sw_if_index,
1472                                 vnet_buffer(p1)->sw_if_index[VLIB_RX]);
1473
1474           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1475           mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
1476
1477           leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
1478
1479           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 0);
1480           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 0);
1481
1482           /* Treat IP frag packets as "experimental" protocol for now
1483              until support of IP frag reassembly is implemented */
1484           proto0 = ip4_is_fragment(ip0) ? 0xfe : ip0->protocol;
1485           proto1 = ip4_is_fragment(ip1) ? 0xfe : ip1->protocol;
1486           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1487           is_udp1 = proto1 == IP_PROTOCOL_UDP;
1488           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1489           is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1490
1491           flags0 = p0->flags;
1492           flags1 = p1->flags;
1493
1494           good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1495           good_tcp_udp1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1496
1497           udp0 = ip4_next_header (ip0);
1498           udp1 = ip4_next_header (ip1);
1499
1500           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1501           good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1502           good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1503
1504           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
1505           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 1);
1506
1507           /* Verify UDP length. */
1508           ip_len0 = clib_net_to_host_u16 (ip0->length);
1509           ip_len1 = clib_net_to_host_u16 (ip1->length);
1510           udp_len0 = clib_net_to_host_u16 (udp0->length);
1511           udp_len1 = clib_net_to_host_u16 (udp1->length);
1512
1513           len_diff0 = ip_len0 - udp_len0;
1514           len_diff1 = ip_len1 - udp_len1;
1515
1516           len_diff0 = is_udp0 ? len_diff0 : 0;
1517           len_diff1 = is_udp1 ? len_diff1 : 0;
1518
1519           if (PREDICT_FALSE (! (is_tcp_udp0 & is_tcp_udp1
1520                                 & good_tcp_udp0 & good_tcp_udp1)))
1521             {
1522               if (is_tcp_udp0)
1523                 {
1524                   if (is_tcp_udp0
1525                       && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1526                     flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
1527                   good_tcp_udp0 =
1528                     (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1529                   good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1530                 }
1531               if (is_tcp_udp1)
1532                 {
1533                   if (is_tcp_udp1
1534                       && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1535                     flags1 = ip4_tcp_udp_validate_checksum (vm, p1);
1536                   good_tcp_udp1 =
1537                     (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1538                   good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1539                 }
1540             }
1541
1542           good_tcp_udp0 &= len_diff0 >= 0;
1543           good_tcp_udp1 &= len_diff1 >= 0;
1544
1545           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1546           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 2);
1547
1548           error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1549
1550           error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
1551           error1 = len_diff1 < 0 ? IP4_ERROR_UDP_LENGTH : error1;
1552
1553           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1554           error0 = (is_tcp_udp0 && ! good_tcp_udp0
1555                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0
1556                     : error0);
1557           error1 = (is_tcp_udp1 && ! good_tcp_udp1
1558                     ? IP4_ERROR_TCP_CHECKSUM + is_udp1
1559                     : error1);
1560
1561           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1562           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 3);
1563           leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
1564           leaf1 = (leaf1 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie1->default_leaf : leaf1);
1565
1566           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1567           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1568
1569           vnet_buffer (p1)->ip.adj_index[VLIB_RX] = lbi1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
1570           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = lbi1;
1571
1572           lb0 = load_balance_get(lbi0);
1573           lb1 = load_balance_get(lbi1);
1574           dpo0 = load_balance_get_bucket_i(lb0, 0);
1575           dpo1 = load_balance_get_bucket_i(lb1, 0);
1576
1577           /*
1578            * Must have a route to source otherwise we drop the packet.
1579            * ip4 broadcasts are accepted, e.g. to make dhcp client work
1580            *
1581            * The checks are:
1582            *  - the source is a recieve => it's from us => bogus, do this
1583            *    first since it sets a different error code.
1584            *  - uRPF check for any route to source - accept if passes.
1585            *  - allow packets destined to the broadcast address from unknown sources
1586            */
1587           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1588                      dpo0->dpoi_type == DPO_RECEIVE) ?
1589                     IP4_ERROR_SPOOFED_LOCAL_PACKETS :
1590                     error0);
1591           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1592                      !fib_urpf_check_size(lb0->lb_urpf) &&
1593                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1594                     ? IP4_ERROR_SRC_LOOKUP_MISS
1595                     : error0);
1596           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1597                      dpo1->dpoi_type == DPO_RECEIVE) ?
1598                     IP4_ERROR_SPOOFED_LOCAL_PACKETS :
1599                     error1);
1600           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1601                      !fib_urpf_check_size(lb1->lb_urpf) &&
1602                      ip1->dst_address.as_u32 != 0xFFFFFFFF)
1603                     ? IP4_ERROR_SRC_LOOKUP_MISS
1604                     : error1);
1605
1606           next0 = lm->local_next_by_ip_protocol[proto0];
1607           next1 = lm->local_next_by_ip_protocol[proto1];
1608
1609           next0 = error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1610           next1 = error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1611
1612           p0->error = error0 ? error_node->errors[error0] : 0;
1613           p1->error = error1 ? error_node->errors[error1] : 0;
1614
1615           enqueue_code = (next0 != next_index) + 2*(next1 != next_index);
1616
1617           if (PREDICT_FALSE (enqueue_code != 0))
1618             {
1619               switch (enqueue_code)
1620                 {
1621                 case 1:
1622                   /* A B A */
1623                   to_next[-2] = pi1;
1624                   to_next -= 1;
1625                   n_left_to_next += 1;
1626                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
1627                   break;
1628
1629                 case 2:
1630                   /* A A B */
1631                   to_next -= 1;
1632                   n_left_to_next += 1;
1633                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
1634                   break;
1635
1636                 case 3:
1637                   /* A B B or A B C */
1638                   to_next -= 2;
1639                   n_left_to_next += 2;
1640                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
1641                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
1642                   if (next0 == next1)
1643                     {
1644                       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1645                       next_index = next1;
1646                       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1647                     }
1648                   break;
1649                 }
1650             }
1651         }
1652
1653       while (n_left_from > 0 && n_left_to_next > 0)
1654         {
1655           vlib_buffer_t * p0;
1656           ip4_header_t * ip0;
1657           udp_header_t * udp0;
1658           ip4_fib_mtrie_t * mtrie0;
1659           ip4_fib_mtrie_leaf_t leaf0;
1660           u32 pi0, next0, ip_len0, udp_len0, flags0, fib_index0, lbi0;
1661           i32 len_diff0;
1662           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1663           load_balance_t *lb0;
1664           const dpo_id_t *dpo0;
1665
1666           pi0 = to_next[0] = from[0];
1667           from += 1;
1668           n_left_from -= 1;
1669           to_next += 1;
1670           n_left_to_next -= 1;
1671
1672           p0 = vlib_get_buffer (vm, pi0);
1673
1674           ip0 = vlib_buffer_get_current (p0);
1675
1676           fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
1677                                 vnet_buffer(p0)->sw_if_index[VLIB_RX]);
1678
1679           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1680
1681           leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
1682
1683           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 0);
1684
1685           /* Treat IP frag packets as "experimental" protocol for now
1686              until support of IP frag reassembly is implemented */
1687           proto0 = ip4_is_fragment(ip0) ? 0xfe : ip0->protocol;
1688           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1689           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1690
1691           flags0 = p0->flags;
1692
1693           good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1694
1695           udp0 = ip4_next_header (ip0);
1696
1697           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1698           good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1699
1700           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
1701
1702           /* Verify UDP length. */
1703           ip_len0 = clib_net_to_host_u16 (ip0->length);
1704           udp_len0 = clib_net_to_host_u16 (udp0->length);
1705
1706           len_diff0 = ip_len0 - udp_len0;
1707
1708           len_diff0 = is_udp0 ? len_diff0 : 0;
1709
1710           if (PREDICT_FALSE (! (is_tcp_udp0 & good_tcp_udp0)))
1711             {
1712               if (is_tcp_udp0)
1713                 {
1714                   if (is_tcp_udp0
1715                       && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1716                     flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
1717                   good_tcp_udp0 =
1718                     (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1719                   good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1720                 }
1721             }
1722
1723           good_tcp_udp0 &= len_diff0 >= 0;
1724
1725           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1726
1727           error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
1728
1729           error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
1730
1731           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1732           error0 = (is_tcp_udp0 && ! good_tcp_udp0
1733                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0
1734                     : error0);
1735
1736           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1737           leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
1738
1739           lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1740           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1741
1742           lb0 = load_balance_get(lbi0);
1743           dpo0 = load_balance_get_bucket_i(lb0, 0);
1744
1745           vnet_buffer (p0)->ip.adj_index[VLIB_TX] =
1746               vnet_buffer (p0)->ip.adj_index[VLIB_RX] =
1747                   dpo0->dpoi_index;
1748
1749           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1750                      dpo0->dpoi_type == DPO_RECEIVE) ?
1751                     IP4_ERROR_SPOOFED_LOCAL_PACKETS :
1752                     error0);
1753           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1754                      !fib_urpf_check_size(lb0->lb_urpf) &&
1755                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1756                     ? IP4_ERROR_SRC_LOOKUP_MISS
1757                     : error0);
1758
1759           next0 = lm->local_next_by_ip_protocol[proto0];
1760
1761           next0 = error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1762
1763           p0->error = error0? error_node->errors[error0] : 0;
1764
1765           if (PREDICT_FALSE (next0 != next_index))
1766             {
1767               n_left_to_next += 1;
1768               vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1769
1770               next_index = next0;
1771               vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1772               to_next[0] = pi0;
1773               to_next += 1;
1774               n_left_to_next -= 1;
1775             }
1776         }
1777
1778       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1779     }
1780
1781   return frame->n_vectors;
1782 }
1783
1784 VLIB_REGISTER_NODE (ip4_local_node,static) = {
1785   .function = ip4_local,
1786   .name = "ip4-local",
1787   .vector_size = sizeof (u32),
1788
1789   .format_trace = format_ip4_forward_next_trace,
1790
1791   .n_next_nodes = IP_LOCAL_N_NEXT,
1792   .next_nodes = {
1793     [IP_LOCAL_NEXT_DROP] = "error-drop",
1794     [IP_LOCAL_NEXT_PUNT] = "error-punt",
1795     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
1796     [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",
1797   },
1798 };
1799
1800 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local)
1801
1802 void ip4_register_protocol (u32 protocol, u32 node_index)
1803 {
1804   vlib_main_t * vm = vlib_get_main();
1805   ip4_main_t * im = &ip4_main;
1806   ip_lookup_main_t * lm = &im->lookup_main;
1807
1808   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1809   lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip4_local_node.index, node_index);
1810 }
1811
1812 static clib_error_t *
1813 show_ip_local_command_fn (vlib_main_t * vm,
1814                           unformat_input_t * input,
1815                          vlib_cli_command_t * cmd)
1816 {
1817   ip4_main_t * im = &ip4_main;
1818   ip_lookup_main_t * lm = &im->lookup_main;
1819   int i;
1820
1821   vlib_cli_output (vm, "Protocols handled by ip4_local");
1822   for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
1823     {
1824       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
1825         vlib_cli_output (vm, "%d", i);
1826     }
1827   return 0;
1828 }
1829
1830
1831
1832 /*?
1833  * Display the set of protocols handled by the local IPv4 stack.
1834  *
1835  * @cliexpar
1836  * Example of how to display local protocol table:
1837  * @cliexstart{show ip local}
1838  * Protocols handled by ip4_local
1839  * 1
1840  * 17
1841  * 47
1842  * @cliexend
1843 ?*/
1844 /* *INDENT-OFF* */
1845 VLIB_CLI_COMMAND (show_ip_local, static) = {
1846   .path = "show ip local",
1847   .function = show_ip_local_command_fn,
1848   .short_help = "show ip local",
1849 };
1850 /* *INDENT-ON* */
1851
1852 always_inline uword
1853 ip4_arp_inline (vlib_main_t * vm,
1854                 vlib_node_runtime_t * node,
1855                 vlib_frame_t * frame,
1856                 int is_glean)
1857 {
1858   vnet_main_t * vnm = vnet_get_main();
1859   ip4_main_t * im = &ip4_main;
1860   ip_lookup_main_t * lm = &im->lookup_main;
1861   u32 * from, * to_next_drop;
1862   uword n_left_from, n_left_to_next_drop, next_index;
1863   static f64 time_last_seed_change = -1e100;
1864   static u32 hash_seeds[3];
1865   static uword hash_bitmap[256 / BITS (uword)];
1866   f64 time_now;
1867
1868   if (node->flags & VLIB_NODE_FLAG_TRACE)
1869     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1870
1871   time_now = vlib_time_now (vm);
1872   if (time_now - time_last_seed_change > 1e-3)
1873     {
1874       uword i;
1875       u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
1876                                              sizeof (hash_seeds));
1877       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1878         hash_seeds[i] = r[i];
1879
1880       /* Mark all hash keys as been no-seen before. */
1881       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1882         hash_bitmap[i] = 0;
1883
1884       time_last_seed_change = time_now;
1885     }
1886
1887   from = vlib_frame_vector_args (frame);
1888   n_left_from = frame->n_vectors;
1889   next_index = node->cached_next_index;
1890   if (next_index == IP4_ARP_NEXT_DROP)
1891     next_index = IP4_ARP_N_NEXT; /* point to first interface */
1892
1893   while (n_left_from > 0)
1894     {
1895       vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
1896                            to_next_drop, n_left_to_next_drop);
1897
1898       while (n_left_from > 0 && n_left_to_next_drop > 0)
1899         {
1900           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
1901           ip_adjacency_t * adj0;
1902           vlib_buffer_t * p0;
1903           ip4_header_t * ip0;
1904           uword bm0;
1905
1906           pi0 = from[0];
1907
1908           p0 = vlib_get_buffer (vm, pi0);
1909
1910           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1911           adj0 = ip_get_adjacency (lm, adj_index0);
1912           ip0 = vlib_buffer_get_current (p0);
1913
1914           a0 = hash_seeds[0];
1915           b0 = hash_seeds[1];
1916           c0 = hash_seeds[2];
1917
1918           sw_if_index0 = adj0->rewrite_header.sw_if_index;
1919           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
1920
1921           if (is_glean)
1922           {
1923               /*
1924                * this is the Glean case, so we are ARPing for the
1925                * packet's destination
1926                */
1927               a0 ^= ip0->dst_address.data_u32;
1928           }
1929           else
1930           {
1931               a0 ^= adj0->sub_type.nbr.next_hop.ip4.data_u32;
1932           }
1933           b0 ^= sw_if_index0;
1934
1935           hash_v3_finalize32 (a0, b0, c0);
1936
1937           c0 &= BITS (hash_bitmap) - 1;
1938           c0 = c0 / BITS (uword);
1939           m0 = (uword) 1 << (c0 % BITS (uword));
1940
1941           bm0 = hash_bitmap[c0];
1942           drop0 = (bm0 & m0) != 0;
1943
1944           /* Mark it as seen. */
1945           hash_bitmap[c0] = bm0 | m0;
1946
1947           from += 1;
1948           n_left_from -= 1;
1949           to_next_drop[0] = pi0;
1950           to_next_drop += 1;
1951           n_left_to_next_drop -= 1;
1952
1953           p0->error = node->errors[drop0 ? IP4_ARP_ERROR_DROP : IP4_ARP_ERROR_REQUEST_SENT];
1954
1955           /*
1956            * the adj has been updated to a rewrite but the node the DPO that got
1957            * us here hasn't - yet. no big deal. we'll drop while we wait.
1958            */
1959           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
1960             continue;
1961
1962           if (drop0)
1963             continue;
1964
1965           /*
1966            * Can happen if the control-plane is programming tables
1967            * with traffic flowing; at least that's today's lame excuse.
1968            */
1969           if ((is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_GLEAN) ||
1970               (!is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP))
1971           {
1972             p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
1973           }
1974           else
1975           /* Send ARP request. */
1976           {
1977             u32 bi0 = 0;
1978             vlib_buffer_t * b0;
1979             ethernet_arp_header_t * h0;
1980             vnet_hw_interface_t * hw_if0;
1981
1982             h0 = vlib_packet_template_get_packet (vm, &im->ip4_arp_request_packet_template, &bi0);
1983
1984             /* Add rewrite/encap string for ARP packet. */
1985             vnet_rewrite_one_header (adj0[0], h0, sizeof (ethernet_header_t));
1986
1987             hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1988
1989             /* Src ethernet address in ARP header. */
1990             clib_memcpy (h0->ip4_over_ethernet[0].ethernet, hw_if0->hw_address,
1991                     sizeof (h0->ip4_over_ethernet[0].ethernet));
1992
1993             if (is_glean)
1994             {
1995                 /* The interface's source address is stashed in the Glean Adj */
1996                 h0->ip4_over_ethernet[0].ip4 = adj0->sub_type.glean.receive_addr.ip4;
1997
1998                 /* Copy in destination address we are requesting. This is the
1999                 * glean case, so it's the packet's destination.*/
2000                 h0->ip4_over_ethernet[1].ip4.data_u32 = ip0->dst_address.data_u32;
2001             }
2002             else
2003             {
2004                 /* Src IP address in ARP header. */
2005                 if (ip4_src_address_for_packet(lm, sw_if_index0,
2006                                                &h0->ip4_over_ethernet[0].ip4))
2007                 {
2008                     /* No source address available */
2009                     p0->error = node->errors[IP4_ARP_ERROR_NO_SOURCE_ADDRESS];
2010                     vlib_buffer_free(vm, &bi0, 1);
2011                     continue;
2012                 }
2013
2014                 /* Copy in destination address we are requesting from the
2015                    incomplete adj */
2016                 h0->ip4_over_ethernet[1].ip4.data_u32 =
2017                     adj0->sub_type.nbr.next_hop.ip4.as_u32;
2018             }
2019
2020             vlib_buffer_copy_trace_flag (vm, p0, bi0);
2021             b0 = vlib_get_buffer (vm, bi0);
2022             vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
2023
2024             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2025
2026             vlib_set_next_frame_buffer (vm, node, adj0->rewrite_header.next_index, bi0);
2027           }
2028         }
2029
2030       vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
2031     }
2032
2033   return frame->n_vectors;
2034 }
2035
2036 static uword
2037 ip4_arp (vlib_main_t * vm,
2038          vlib_node_runtime_t * node,
2039          vlib_frame_t * frame)
2040 {
2041     return (ip4_arp_inline(vm, node, frame, 0));
2042 }
2043
2044 static uword
2045 ip4_glean (vlib_main_t * vm,
2046            vlib_node_runtime_t * node,
2047            vlib_frame_t * frame)
2048 {
2049     return (ip4_arp_inline(vm, node, frame, 1));
2050 }
2051
2052 static char * ip4_arp_error_strings[] = {
2053   [IP4_ARP_ERROR_DROP] = "address overflow drops",
2054   [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
2055   [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
2056   [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
2057   [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
2058   [IP4_ARP_ERROR_NO_SOURCE_ADDRESS] = "no source address for ARP request",
2059 };
2060
2061 VLIB_REGISTER_NODE (ip4_arp_node) = {
2062   .function = ip4_arp,
2063   .name = "ip4-arp",
2064   .vector_size = sizeof (u32),
2065
2066   .format_trace = format_ip4_forward_next_trace,
2067
2068   .n_errors = ARRAY_LEN (ip4_arp_error_strings),
2069   .error_strings = ip4_arp_error_strings,
2070
2071   .n_next_nodes = IP4_ARP_N_NEXT,
2072   .next_nodes = {
2073     [IP4_ARP_NEXT_DROP] = "error-drop",
2074   },
2075 };
2076
2077 VLIB_REGISTER_NODE (ip4_glean_node) = {
2078   .function = ip4_glean,
2079   .name = "ip4-glean",
2080   .vector_size = sizeof (u32),
2081
2082   .format_trace = format_ip4_forward_next_trace,
2083
2084   .n_errors = ARRAY_LEN (ip4_arp_error_strings),
2085   .error_strings = ip4_arp_error_strings,
2086
2087   .n_next_nodes = IP4_ARP_N_NEXT,
2088   .next_nodes = {
2089     [IP4_ARP_NEXT_DROP] = "error-drop",
2090   },
2091 };
2092
2093 #define foreach_notrace_ip4_arp_error           \
2094 _(DROP)                                         \
2095 _(REQUEST_SENT)                                 \
2096 _(REPLICATE_DROP)                               \
2097 _(REPLICATE_FAIL)
2098
2099 clib_error_t * arp_notrace_init (vlib_main_t * vm)
2100 {
2101   vlib_node_runtime_t *rt =
2102     vlib_node_get_runtime (vm, ip4_arp_node.index);
2103
2104   /* don't trace ARP request packets */
2105 #define _(a)                                    \
2106     vnet_pcap_drop_trace_filter_add_del         \
2107         (rt->errors[IP4_ARP_ERROR_##a],         \
2108          1 /* is_add */);
2109     foreach_notrace_ip4_arp_error;
2110 #undef _
2111   return 0;
2112 }
2113
2114 VLIB_INIT_FUNCTION(arp_notrace_init);
2115
2116
2117 /* Send an ARP request to see if given destination is reachable on given interface. */
2118 clib_error_t *
2119 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
2120 {
2121   vnet_main_t * vnm = vnet_get_main();
2122   ip4_main_t * im = &ip4_main;
2123   ethernet_arp_header_t * h;
2124   ip4_address_t * src;
2125   ip_interface_address_t * ia;
2126   ip_adjacency_t * adj;
2127   vnet_hw_interface_t * hi;
2128   vnet_sw_interface_t * si;
2129   vlib_buffer_t * b;
2130   u32 bi = 0;
2131
2132   si = vnet_get_sw_interface (vnm, sw_if_index);
2133
2134   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2135     {
2136       return clib_error_return (0, "%U: interface %U down",
2137                                 format_ip4_address, dst,
2138                                 format_vnet_sw_if_index_name, vnm,
2139                                 sw_if_index);
2140     }
2141
2142   src = ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2143   if (! src)
2144     {
2145       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2146       return clib_error_return
2147         (0, "no matching interface address for destination %U (interface %U)",
2148          format_ip4_address, dst,
2149          format_vnet_sw_if_index_name, vnm, sw_if_index);
2150     }
2151
2152   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
2153
2154   h = vlib_packet_template_get_packet (vm, &im->ip4_arp_request_packet_template, &bi);
2155
2156   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2157
2158   clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address, sizeof (h->ip4_over_ethernet[0].ethernet));
2159
2160   h->ip4_over_ethernet[0].ip4 = src[0];
2161   h->ip4_over_ethernet[1].ip4 = dst[0];
2162
2163   b = vlib_get_buffer (vm, bi);
2164   vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2165
2166   /* Add encapsulation string for software interface (e.g. ethernet header). */
2167   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2168   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2169
2170   {
2171     vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
2172     u32 * to_next = vlib_frame_vector_args (f);
2173     to_next[0] = bi;
2174     f->n_vectors = 1;
2175     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2176   }
2177
2178   return /* no error */ 0;
2179 }
2180
2181 typedef enum {
2182   IP4_REWRITE_NEXT_DROP,
2183   IP4_REWRITE_NEXT_ARP,
2184   IP4_REWRITE_NEXT_ICMP_ERROR,
2185 } ip4_rewrite_next_t;
2186
2187 always_inline uword
2188 ip4_rewrite_inline (vlib_main_t * vm,
2189                     vlib_node_runtime_t * node,
2190                     vlib_frame_t * frame,
2191                     int rewrite_for_locally_received_packets,
2192                     int is_midchain)
2193 {
2194   ip_lookup_main_t * lm = &ip4_main.lookup_main;
2195   u32 * from = vlib_frame_vector_args (frame);
2196   u32 n_left_from, n_left_to_next, * to_next, next_index;
2197   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip4_input_node.index);
2198   vlib_rx_or_tx_t adj_rx_tx = rewrite_for_locally_received_packets ? VLIB_RX : VLIB_TX;
2199
2200   n_left_from = frame->n_vectors;
2201   next_index = node->cached_next_index;
2202   u32 cpu_index = os_get_cpu_number();
2203
2204   while (n_left_from > 0)
2205     {
2206       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2207
2208       while (n_left_from >= 4 && n_left_to_next >= 2)
2209         {
2210           ip_adjacency_t * adj0, * adj1;
2211           vlib_buffer_t * p0, * p1;
2212           ip4_header_t * ip0, * ip1;
2213           u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
2214           u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
2215           u32 next0_override, next1_override;
2216           u32 tx_sw_if_index0, tx_sw_if_index1;
2217
2218           if (rewrite_for_locally_received_packets)
2219               next0_override = next1_override = 0;
2220
2221           /* Prefetch next iteration. */
2222           {
2223             vlib_buffer_t * p2, * p3;
2224
2225             p2 = vlib_get_buffer (vm, from[2]);
2226             p3 = vlib_get_buffer (vm, from[3]);
2227
2228             vlib_prefetch_buffer_header (p2, STORE);
2229             vlib_prefetch_buffer_header (p3, STORE);
2230
2231             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2232             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2233           }
2234
2235           pi0 = to_next[0] = from[0];
2236           pi1 = to_next[1] = from[1];
2237
2238           from += 2;
2239           n_left_from -= 2;
2240           to_next += 2;
2241           n_left_to_next -= 2;
2242
2243           p0 = vlib_get_buffer (vm, pi0);
2244           p1 = vlib_get_buffer (vm, pi1);
2245
2246           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2247           adj_index1 = vnet_buffer (p1)->ip.adj_index[adj_rx_tx];
2248
2249           /* We should never rewrite a pkt using the MISS adjacency */
2250           ASSERT(adj_index0 && adj_index1);
2251
2252           ip0 = vlib_buffer_get_current (p0);
2253           ip1 = vlib_buffer_get_current (p1);
2254
2255           error0 = error1 = IP4_ERROR_NONE;
2256           next0 = next1 = IP4_REWRITE_NEXT_DROP;
2257
2258           /* Decrement TTL & update checksum.
2259              Works either endian, so no need for byte swap. */
2260           if (! rewrite_for_locally_received_packets)
2261             {
2262               i32 ttl0 = ip0->ttl, ttl1 = ip1->ttl;
2263
2264               /* Input node should have reject packets with ttl 0. */
2265               ASSERT (ip0->ttl > 0);
2266               ASSERT (ip1->ttl > 0);
2267
2268               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2269               checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2270
2271               checksum0 += checksum0 >= 0xffff;
2272               checksum1 += checksum1 >= 0xffff;
2273
2274               ip0->checksum = checksum0;
2275               ip1->checksum = checksum1;
2276
2277               ttl0 -= 1;
2278               ttl1 -= 1;
2279
2280               ip0->ttl = ttl0;
2281               ip1->ttl = ttl1;
2282
2283               /*
2284                * If the ttl drops below 1 when forwarding, generate
2285                * an ICMP response.
2286                */
2287               if (PREDICT_FALSE(ttl0 <= 0))
2288                 {
2289                   error0 = IP4_ERROR_TIME_EXPIRED;
2290                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2291                   icmp4_error_set_vnet_buffer(p0, ICMP4_time_exceeded,
2292                               ICMP4_time_exceeded_ttl_exceeded_in_transit, 0);
2293                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2294                 }
2295               if (PREDICT_FALSE(ttl1 <= 0))
2296                 {
2297                   error1 = IP4_ERROR_TIME_EXPIRED;
2298                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32)~0;
2299                   icmp4_error_set_vnet_buffer(p1, ICMP4_time_exceeded,
2300                               ICMP4_time_exceeded_ttl_exceeded_in_transit, 0);
2301                   next1 = IP4_REWRITE_NEXT_ICMP_ERROR;
2302                 }
2303
2304               /* Verify checksum. */
2305               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2306               ASSERT (ip1->checksum == ip4_header_checksum (ip1));
2307             }
2308
2309           /* Rewrite packet header and updates lengths. */
2310           adj0 = ip_get_adjacency (lm, adj_index0);
2311           adj1 = ip_get_adjacency (lm, adj_index1);
2312
2313           if (rewrite_for_locally_received_packets)
2314             {
2315               if (PREDICT_FALSE(adj0->lookup_next_index
2316                                 == IP_LOOKUP_NEXT_ARP))
2317                 next0_override = IP4_REWRITE_NEXT_ARP;
2318               if (PREDICT_FALSE(adj1->lookup_next_index
2319                                 == IP_LOOKUP_NEXT_ARP))
2320                 next1_override = IP4_REWRITE_NEXT_ARP;
2321             }
2322
2323           /* Worth pipelining. No guarantee that adj0,1 are hot... */
2324           rw_len0 = adj0[0].rewrite_header.data_bytes;
2325           rw_len1 = adj1[0].rewrite_header.data_bytes;
2326           vnet_buffer(p0)->ip.save_rewrite_length = rw_len0;
2327           vnet_buffer(p1)->ip.save_rewrite_length = rw_len1;
2328
2329           /* Check MTU of outgoing interface. */
2330           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2331                     ? IP4_ERROR_MTU_EXCEEDED
2332                     : error0);
2333           error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
2334                     ? IP4_ERROR_MTU_EXCEEDED
2335                     : error1);
2336
2337           next0 = (error0 == IP4_ERROR_NONE)
2338             ? adj0[0].rewrite_header.next_index : next0;
2339
2340           if (rewrite_for_locally_received_packets)
2341               next0 = next0 && next0_override ? next0_override : next0;
2342
2343           next1 = (error1 == IP4_ERROR_NONE)
2344             ? adj1[0].rewrite_header.next_index : next1;
2345
2346           if (rewrite_for_locally_received_packets)
2347               next1 = next1 && next1_override ? next1_override : next1;
2348
2349           /*
2350            * We've already accounted for an ethernet_header_t elsewhere
2351            */
2352           if (PREDICT_FALSE (rw_len0 > sizeof(ethernet_header_t)))
2353               vlib_increment_combined_counter
2354                   (&adjacency_counters,
2355                    cpu_index, adj_index0,
2356                    /* packet increment */ 0,
2357                    /* byte increment */ rw_len0-sizeof(ethernet_header_t));
2358
2359           if (PREDICT_FALSE (rw_len1 > sizeof(ethernet_header_t)))
2360               vlib_increment_combined_counter
2361                   (&adjacency_counters,
2362                    cpu_index, adj_index1,
2363                    /* packet increment */ 0,
2364                    /* byte increment */ rw_len1-sizeof(ethernet_header_t));
2365
2366           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2367            * to see the IP headerr */
2368           if (PREDICT_TRUE(error0 == IP4_ERROR_NONE))
2369             {
2370               p0->current_data -= rw_len0;
2371               p0->current_length += rw_len0;
2372               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2373               vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2374                   tx_sw_if_index0;
2375
2376               vnet_feature_arc_start(lm->output_feature_arc_index,
2377                                      tx_sw_if_index0, &next0, p0);
2378             }
2379           if (PREDICT_TRUE(error1 == IP4_ERROR_NONE))
2380             {
2381               p1->current_data -= rw_len1;
2382               p1->current_length += rw_len1;
2383
2384               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
2385               vnet_buffer (p1)->sw_if_index[VLIB_TX] =
2386                   tx_sw_if_index1;
2387
2388               vnet_feature_arc_start(lm->output_feature_arc_index,
2389                                      tx_sw_if_index1, &next1, p1);
2390             }
2391
2392           /* Guess we are only writing on simple Ethernet header. */
2393           vnet_rewrite_two_headers (adj0[0], adj1[0],
2394                                     ip0, ip1,
2395                                     sizeof (ethernet_header_t));
2396
2397           if (is_midchain)
2398           {
2399               adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
2400               adj1->sub_type.midchain.fixup_func(vm, adj1, p1);
2401           }
2402
2403           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2404                                            to_next, n_left_to_next,
2405                                            pi0, pi1, next0, next1);
2406         }
2407
2408       while (n_left_from > 0 && n_left_to_next > 0)
2409         {
2410           ip_adjacency_t * adj0;
2411           vlib_buffer_t * p0;
2412           ip4_header_t * ip0;
2413           u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2414           u32 next0_override;
2415           u32 tx_sw_if_index0;
2416
2417           if (rewrite_for_locally_received_packets)
2418               next0_override = 0;
2419
2420           pi0 = to_next[0] = from[0];
2421
2422           p0 = vlib_get_buffer (vm, pi0);
2423
2424           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2425
2426           /* We should never rewrite a pkt using the MISS adjacency */
2427           ASSERT(adj_index0);
2428
2429           adj0 = ip_get_adjacency (lm, adj_index0);
2430
2431           ip0 = vlib_buffer_get_current (p0);
2432
2433           error0 = IP4_ERROR_NONE;
2434           next0 = IP4_REWRITE_NEXT_DROP;            /* drop on error */
2435
2436           /* Decrement TTL & update checksum. */
2437           if (! rewrite_for_locally_received_packets)
2438             {
2439               i32 ttl0 = ip0->ttl;
2440
2441               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2442
2443               checksum0 += checksum0 >= 0xffff;
2444
2445               ip0->checksum = checksum0;
2446
2447               ASSERT (ip0->ttl > 0);
2448
2449               ttl0 -= 1;
2450
2451               ip0->ttl = ttl0;
2452
2453               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2454
2455               if (PREDICT_FALSE(ttl0 <= 0))
2456                 {
2457                   /*
2458                    * If the ttl drops below 1 when forwarding, generate
2459                    * an ICMP response.
2460                    */
2461                   error0 = IP4_ERROR_TIME_EXPIRED;
2462                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2463                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2464                   icmp4_error_set_vnet_buffer(p0, ICMP4_time_exceeded,
2465                               ICMP4_time_exceeded_ttl_exceeded_in_transit, 0);
2466                 }
2467             }
2468
2469           if (rewrite_for_locally_received_packets)
2470             {
2471               /*
2472                * We have to override the next_index in ARP adjacencies,
2473                * because they're set up for ip4-arp, not this node...
2474                */
2475               if (PREDICT_FALSE(adj0->lookup_next_index
2476                                 == IP_LOOKUP_NEXT_ARP))
2477                 next0_override = IP4_REWRITE_NEXT_ARP;
2478             }
2479
2480           /* Guess we are only writing on simple Ethernet header. */
2481           vnet_rewrite_one_header (adj0[0], ip0,
2482                                    sizeof (ethernet_header_t));
2483
2484           /* Update packet buffer attributes/set output interface. */
2485           rw_len0 = adj0[0].rewrite_header.data_bytes;
2486           vnet_buffer(p0)->ip.save_rewrite_length = rw_len0;
2487
2488           if (PREDICT_FALSE (rw_len0 > sizeof(ethernet_header_t)))
2489               vlib_increment_combined_counter
2490                   (&adjacency_counters,
2491                    cpu_index, adj_index0,
2492                    /* packet increment */ 0,
2493                    /* byte increment */ rw_len0-sizeof(ethernet_header_t));
2494
2495           /* Check MTU of outgoing interface. */
2496           error0 = (vlib_buffer_length_in_chain (vm, p0)
2497                     > adj0[0].rewrite_header.max_l3_packet_bytes
2498                     ? IP4_ERROR_MTU_EXCEEDED
2499                     : error0);
2500
2501           p0->error = error_node->errors[error0];
2502
2503           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2504            * to see the IP headerr */
2505           if (PREDICT_TRUE(error0 == IP4_ERROR_NONE))
2506             {
2507               p0->current_data -= rw_len0;
2508               p0->current_length += rw_len0;
2509               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2510
2511               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2512               next0 = adj0[0].rewrite_header.next_index;
2513
2514               if (is_midchain)
2515                 {
2516                   adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
2517                 }
2518
2519               vnet_feature_arc_start(lm->output_feature_arc_index,
2520                                      tx_sw_if_index0, &next0, p0);
2521
2522             }
2523
2524           if (rewrite_for_locally_received_packets)
2525               next0 = next0 && next0_override ? next0_override : next0;
2526
2527           from += 1;
2528           n_left_from -= 1;
2529           to_next += 1;
2530           n_left_to_next -= 1;
2531
2532           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2533                                            to_next, n_left_to_next,
2534                                            pi0, next0);
2535         }
2536
2537       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2538     }
2539
2540   /* Need to do trace after rewrites to pick up new packet data. */
2541   if (node->flags & VLIB_NODE_FLAG_TRACE)
2542     ip4_forward_next_trace (vm, node, frame, adj_rx_tx);
2543
2544   return frame->n_vectors;
2545 }
2546
2547
2548 /** @brief IPv4 transit rewrite node.
2549     @node ip4-rewrite-transit
2550
2551     This is the IPv4 transit-rewrite node: decrement TTL, fix the ipv4
2552     header checksum, fetch the ip adjacency, check the outbound mtu,
2553     apply the adjacency rewrite, and send pkts to the adjacency
2554     rewrite header's rewrite_next_index.
2555
2556     @param vm vlib_main_t corresponding to the current thread
2557     @param node vlib_node_runtime_t
2558     @param frame vlib_frame_t whose contents should be dispatched
2559
2560     @par Graph mechanics: buffer metadata, next index usage
2561
2562     @em Uses:
2563     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
2564         - the rewrite adjacency index
2565     - <code>adj->lookup_next_index</code>
2566         - Must be IP_LOOKUP_NEXT_REWRITE or IP_LOOKUP_NEXT_ARP, otherwise
2567           the packet will be dropped.
2568     - <code>adj->rewrite_header</code>
2569         - Rewrite string length, rewrite string, next_index
2570
2571     @em Sets:
2572     - <code>b->current_data, b->current_length</code>
2573         - Updated net of applying the rewrite string
2574
2575     <em>Next Indices:</em>
2576     - <code> adj->rewrite_header.next_index </code>
2577       or @c error-drop
2578 */
2579 static uword
2580 ip4_rewrite_transit (vlib_main_t * vm,
2581                      vlib_node_runtime_t * node,
2582                      vlib_frame_t * frame)
2583 {
2584   return ip4_rewrite_inline (vm, node, frame,
2585                              /* rewrite_for_locally_received_packets */ 0, 0);
2586 }
2587
2588 /** @brief IPv4 local rewrite node.
2589     @node ip4-rewrite-local
2590
2591     This is the IPv4 local rewrite node. Fetch the ip adjacency, check
2592     the outbound interface mtu, apply the adjacency rewrite, and send
2593     pkts to the adjacency rewrite header's rewrite_next_index. Deal
2594     with hemorrhoids of the form "some clown sends an icmp4 w/ src =
2595     dst = interface addr."
2596
2597     @param vm vlib_main_t corresponding to the current thread
2598     @param node vlib_node_runtime_t
2599     @param frame vlib_frame_t whose contents should be dispatched
2600
2601     @par Graph mechanics: buffer metadata, next index usage
2602
2603     @em Uses:
2604     - <code>vnet_buffer(b)->ip.adj_index[VLIB_RX]</code>
2605         - the rewrite adjacency index
2606     - <code>adj->lookup_next_index</code>
2607         - Must be IP_LOOKUP_NEXT_REWRITE or IP_LOOKUP_NEXT_ARP, otherwise
2608           the packet will be dropped.
2609     - <code>adj->rewrite_header</code>
2610         - Rewrite string length, rewrite string, next_index
2611
2612     @em Sets:
2613     - <code>b->current_data, b->current_length</code>
2614         - Updated net of applying the rewrite string
2615
2616     <em>Next Indices:</em>
2617     - <code> adj->rewrite_header.next_index </code>
2618       or @c error-drop
2619 */
2620
2621 static uword
2622 ip4_rewrite_local (vlib_main_t * vm,
2623                    vlib_node_runtime_t * node,
2624                    vlib_frame_t * frame)
2625 {
2626   return ip4_rewrite_inline (vm, node, frame,
2627                              /* rewrite_for_locally_received_packets */ 1, 0);
2628 }
2629
2630 static uword
2631 ip4_midchain (vlib_main_t * vm,
2632               vlib_node_runtime_t * node,
2633               vlib_frame_t * frame)
2634 {
2635   return ip4_rewrite_inline (vm, node, frame,
2636                              /* rewrite_for_locally_received_packets */ 0, 1);
2637 }
2638
2639 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2640   .function = ip4_rewrite_transit,
2641   .name = "ip4-rewrite-transit",
2642   .vector_size = sizeof (u32),
2643
2644   .format_trace = format_ip4_rewrite_trace,
2645
2646   .n_next_nodes = 3,
2647   .next_nodes = {
2648     [IP4_REWRITE_NEXT_DROP] = "error-drop",
2649     [IP4_REWRITE_NEXT_ARP] = "ip4-arp",
2650     [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2651   },
2652 };
2653
2654 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite_transit)
2655
2656 VLIB_REGISTER_NODE (ip4_midchain_node) = {
2657   .function = ip4_midchain,
2658   .name = "ip4-midchain",
2659   .vector_size = sizeof (u32),
2660
2661   .format_trace = format_ip4_forward_next_trace,
2662
2663   .sibling_of = "ip4-rewrite-transit",
2664 };
2665
2666 VLIB_NODE_FUNCTION_MULTIARCH (ip4_midchain_node, ip4_midchain)
2667
2668 VLIB_REGISTER_NODE (ip4_rewrite_local_node) = {
2669   .function = ip4_rewrite_local,
2670   .name = "ip4-rewrite-local",
2671   .vector_size = sizeof (u32),
2672
2673   .sibling_of = "ip4-rewrite-transit",
2674
2675   .format_trace = format_ip4_rewrite_trace,
2676
2677   .n_next_nodes = 0,
2678 };
2679
2680 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_local_node, ip4_rewrite_local)
2681
2682 static clib_error_t *
2683 add_del_interface_table (vlib_main_t * vm,
2684                          unformat_input_t * input,
2685                          vlib_cli_command_t * cmd)
2686 {
2687   vnet_main_t * vnm = vnet_get_main();
2688   clib_error_t * error = 0;
2689   u32 sw_if_index, table_id;
2690
2691   sw_if_index = ~0;
2692
2693   if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2694     {
2695       error = clib_error_return (0, "unknown interface `%U'",
2696                                  format_unformat_error, input);
2697       goto done;
2698     }
2699
2700   if (unformat (input, "%d", &table_id))
2701     ;
2702   else
2703     {
2704       error = clib_error_return (0, "expected table id `%U'",
2705                                  format_unformat_error, input);
2706       goto done;
2707     }
2708
2709   {
2710     ip4_main_t * im = &ip4_main;
2711     u32 fib_index;
2712
2713     fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2714                                                    table_id);
2715
2716     //
2717     // FIXME-LATER
2718     //  changing an interface's table has consequences for any connecteds
2719     //  and adj-fibs already installed.
2720     //
2721     vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
2722     im->fib_index_by_sw_if_index[sw_if_index] = fib_index;
2723   }
2724
2725  done:
2726   return error;
2727 }
2728
2729 /*?
2730  * Place the indicated interface into the supplied IPv4 FIB table (also known
2731  * as a VRF). If the FIB table does not exist, this command creates it. To
2732  * display the current IPv4 FIB table, use the command '<em>show ip fib</em>'.
2733  * FIB table will only be displayed if a route has been added to the table, or
2734  * an IP Address is assigned to an interface in the table (which adds a route
2735  * automatically).
2736  *
2737  * @note IP addresses added after setting the interface IP table end up in
2738  * the indicated FIB table. If the IP address is added prior to adding the
2739  * interface to the FIB table, it will NOT be part of the FIB table. Predictable
2740  * but potentially counter-intuitive results occur if you provision interface
2741  * addresses in multiple FIBs. Upon RX, packets will be processed in the last
2742  * IP table ID provisioned. It might be marginally useful to evade source RPF
2743  * drops to put an interface address into multiple FIBs.
2744  *
2745  * @cliexpar
2746  * Example of how to add an interface to an IPv4 FIB table (where 2 is the table-id):
2747  * @cliexcmd{set interface ip table GigabitEthernet2/0/0 2}
2748  ?*/
2749 /* *INDENT-OFF* */
2750 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) = {
2751   .path = "set interface ip table",
2752   .function = add_del_interface_table,
2753   .short_help = "set interface ip table <interface> <table-id>",
2754 };
2755 /* *INDENT-ON* */
2756
2757
2758 static uword
2759 ip4_lookup_multicast (vlib_main_t * vm,
2760                       vlib_node_runtime_t * node,
2761                       vlib_frame_t * frame)
2762 {
2763   ip4_main_t * im = &ip4_main;
2764   vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
2765   u32 n_left_from, n_left_to_next, * from, * to_next;
2766   ip_lookup_next_t next;
2767   u32 cpu_index = os_get_cpu_number();
2768
2769   from = vlib_frame_vector_args (frame);
2770   n_left_from = frame->n_vectors;
2771   next = node->cached_next_index;
2772
2773   while (n_left_from > 0)
2774     {
2775       vlib_get_next_frame (vm, node, next,
2776                            to_next, n_left_to_next);
2777
2778       while (n_left_from >= 4 && n_left_to_next >= 2)
2779         {
2780           vlib_buffer_t * p0, * p1;
2781           u32 pi0, pi1, lb_index0, lb_index1, wrong_next;
2782           ip_lookup_next_t next0, next1;
2783           ip4_header_t * ip0, * ip1;
2784           u32 fib_index0, fib_index1;
2785           const dpo_id_t *dpo0, *dpo1;
2786           const load_balance_t * lb0, * lb1;
2787
2788           /* Prefetch next iteration. */
2789           {
2790             vlib_buffer_t * p2, * p3;
2791
2792             p2 = vlib_get_buffer (vm, from[2]);
2793             p3 = vlib_get_buffer (vm, from[3]);
2794
2795             vlib_prefetch_buffer_header (p2, LOAD);
2796             vlib_prefetch_buffer_header (p3, LOAD);
2797
2798             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
2799             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
2800           }
2801
2802           pi0 = to_next[0] = from[0];
2803           pi1 = to_next[1] = from[1];
2804
2805           p0 = vlib_get_buffer (vm, pi0);
2806           p1 = vlib_get_buffer (vm, pi1);
2807
2808           ip0 = vlib_buffer_get_current (p0);
2809           ip1 = vlib_buffer_get_current (p1);
2810
2811           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
2812           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
2813           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
2814             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
2815           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
2816             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
2817
2818           lb_index0 = ip4_fib_table_lookup_lb (ip4_fib_get(fib_index0),
2819                                                &ip0->dst_address);
2820           lb_index1 = ip4_fib_table_lookup_lb (ip4_fib_get(fib_index1),
2821                                                &ip1->dst_address);
2822
2823           lb0 = load_balance_get (lb_index0);
2824           lb1 = load_balance_get (lb_index1);
2825
2826           ASSERT (lb0->lb_n_buckets > 0);
2827           ASSERT (is_pow2 (lb0->lb_n_buckets));
2828           ASSERT (lb1->lb_n_buckets > 0);
2829           ASSERT (is_pow2 (lb1->lb_n_buckets));
2830
2831           vnet_buffer (p0)->ip.flow_hash = ip4_compute_flow_hash
2832               (ip0, lb0->lb_hash_config);
2833
2834           vnet_buffer (p1)->ip.flow_hash = ip4_compute_flow_hash
2835               (ip1, lb1->lb_hash_config);
2836
2837           dpo0 = load_balance_get_bucket_i(lb0,
2838                                            (vnet_buffer (p0)->ip.flow_hash &
2839                                             (lb0->lb_n_buckets_minus_1)));
2840           dpo1 = load_balance_get_bucket_i(lb1,
2841                                            (vnet_buffer (p1)->ip.flow_hash &
2842                                             (lb1->lb_n_buckets_minus_1)));
2843
2844           next0 = dpo0->dpoi_next_node;
2845           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
2846           next1 = dpo1->dpoi_next_node;
2847           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
2848
2849           if (1) /* $$$$$$ HACK FIXME */
2850           vlib_increment_combined_counter
2851               (cm, cpu_index, lb_index0, 1,
2852                vlib_buffer_length_in_chain (vm, p0));
2853           if (1) /* $$$$$$ HACK FIXME */
2854           vlib_increment_combined_counter
2855               (cm, cpu_index, lb_index1, 1,
2856                vlib_buffer_length_in_chain (vm, p1));
2857
2858           from += 2;
2859           to_next += 2;
2860           n_left_to_next -= 2;
2861           n_left_from -= 2;
2862
2863           wrong_next = (next0 != next) + 2*(next1 != next);
2864           if (PREDICT_FALSE (wrong_next != 0))
2865             {
2866               switch (wrong_next)
2867                 {
2868                 case 1:
2869                   /* A B A */
2870                   to_next[-2] = pi1;
2871                   to_next -= 1;
2872                   n_left_to_next += 1;
2873                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
2874                   break;
2875
2876                 case 2:
2877                   /* A A B */
2878                   to_next -= 1;
2879                   n_left_to_next += 1;
2880                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
2881                   break;
2882
2883                 case 3:
2884                   /* A B C */
2885                   to_next -= 2;
2886                   n_left_to_next += 2;
2887                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
2888                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
2889                   if (next0 == next1)
2890                     {
2891                       /* A B B */
2892                       vlib_put_next_frame (vm, node, next, n_left_to_next);
2893                       next = next1;
2894                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
2895                     }
2896                 }
2897             }
2898         }
2899
2900       while (n_left_from > 0 && n_left_to_next > 0)
2901         {
2902           vlib_buffer_t * p0;
2903           ip4_header_t * ip0;
2904           u32 pi0, lb_index0;
2905           ip_lookup_next_t next0;
2906           u32 fib_index0;
2907           const dpo_id_t *dpo0;
2908           const load_balance_t * lb0;
2909
2910           pi0 = from[0];
2911           to_next[0] = pi0;
2912
2913           p0 = vlib_get_buffer (vm, pi0);
2914
2915           ip0 = vlib_buffer_get_current (p0);
2916
2917           fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
2918                                 vnet_buffer (p0)->sw_if_index[VLIB_RX]);
2919           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
2920               fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
2921
2922           lb_index0 = ip4_fib_table_lookup_lb (ip4_fib_get(fib_index0),
2923                                                &ip0->dst_address);
2924
2925           lb0 = load_balance_get (lb_index0);
2926
2927           ASSERT (lb0->lb_n_buckets > 0);
2928           ASSERT (is_pow2 (lb0->lb_n_buckets));
2929
2930           vnet_buffer (p0)->ip.flow_hash = ip4_compute_flow_hash
2931               (ip0, lb0->lb_hash_config);
2932
2933           dpo0 = load_balance_get_bucket_i(lb0,
2934                                            (vnet_buffer (p0)->ip.flow_hash &
2935                                             (lb0->lb_n_buckets_minus_1)));
2936
2937           next0 = dpo0->dpoi_next_node;
2938           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
2939
2940           if (1) /* $$$$$$ HACK FIXME */
2941               vlib_increment_combined_counter
2942                   (cm, cpu_index, lb_index0, 1,
2943                    vlib_buffer_length_in_chain (vm, p0));
2944
2945           from += 1;
2946           to_next += 1;
2947           n_left_to_next -= 1;
2948           n_left_from -= 1;
2949
2950           if (PREDICT_FALSE (next0 != next))
2951             {
2952               n_left_to_next += 1;
2953               vlib_put_next_frame (vm, node, next, n_left_to_next);
2954               next = next0;
2955               vlib_get_next_frame (vm, node, next,
2956                                    to_next, n_left_to_next);
2957               to_next[0] = pi0;
2958               to_next += 1;
2959               n_left_to_next -= 1;
2960             }
2961         }
2962
2963       vlib_put_next_frame (vm, node, next, n_left_to_next);
2964     }
2965
2966   if (node->flags & VLIB_NODE_FLAG_TRACE)
2967       ip4_forward_next_trace(vm, node, frame, VLIB_TX);
2968
2969   return frame->n_vectors;
2970 }
2971
2972 VLIB_REGISTER_NODE (ip4_lookup_multicast_node,static) = {
2973   .function = ip4_lookup_multicast,
2974   .name = "ip4-lookup-multicast",
2975   .vector_size = sizeof (u32),
2976   .sibling_of = "ip4-lookup",
2977   .format_trace = format_ip4_lookup_trace,
2978
2979   .n_next_nodes = 0,
2980 };
2981
2982 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_multicast_node, ip4_lookup_multicast)
2983
2984 VLIB_REGISTER_NODE (ip4_multicast_node,static) = {
2985   .function = ip4_drop,
2986   .name = "ip4-multicast",
2987   .vector_size = sizeof (u32),
2988
2989   .format_trace = format_ip4_forward_next_trace,
2990
2991   .n_next_nodes = 1,
2992   .next_nodes = {
2993     [0] = "error-drop",
2994   },
2995 };
2996
2997 int ip4_lookup_validate (ip4_address_t *a, u32 fib_index0)
2998 {
2999   ip4_fib_mtrie_t * mtrie0;
3000   ip4_fib_mtrie_leaf_t leaf0;
3001   u32 lbi0;
3002
3003   mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
3004
3005   leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
3006   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 0);
3007   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 1);
3008   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
3009   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
3010
3011   /* Handle default route. */
3012   leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
3013
3014   lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
3015
3016   return lbi0 == ip4_fib_table_lookup_lb (ip4_fib_get(fib_index0), a);
3017 }
3018
3019 static clib_error_t *
3020 test_lookup_command_fn (vlib_main_t * vm,
3021                         unformat_input_t * input,
3022                         vlib_cli_command_t * cmd)
3023 {
3024   ip4_fib_t *fib;
3025   u32 table_id = 0;
3026   f64 count = 1;
3027   u32 n;
3028   int i;
3029   ip4_address_t ip4_base_address;
3030   u64 errors = 0;
3031
3032   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3033       if (unformat (input, "table %d", &table_id))
3034       {
3035           /* Make sure the entry exists. */
3036           fib = ip4_fib_get(table_id);
3037           if ((fib) && (fib->index != table_id))
3038               return clib_error_return (0, "<fib-index> %d does not exist",
3039                                         table_id);
3040       }
3041       else if (unformat (input, "count %f", &count))
3042         ;
3043
3044       else if (unformat (input, "%U",
3045                          unformat_ip4_address, &ip4_base_address))
3046         ;
3047       else
3048         return clib_error_return (0, "unknown input `%U'",
3049                                   format_unformat_error, input);
3050   }
3051
3052   n = count;
3053
3054   for (i = 0; i < n; i++)
3055     {
3056       if (!ip4_lookup_validate (&ip4_base_address, table_id))
3057         errors++;
3058
3059       ip4_base_address.as_u32 =
3060         clib_host_to_net_u32 (1 +
3061                               clib_net_to_host_u32 (ip4_base_address.as_u32));
3062     }
3063
3064   if (errors)
3065     vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
3066   else
3067     vlib_cli_output (vm, "No errors in %d lookups\n", n);
3068
3069   return 0;
3070 }
3071
3072 /*?
3073  * Perform a lookup of an IPv4 Address (or range of addresses) in the
3074  * given FIB table to determine if there is a conflict with the
3075  * adjacency table. The fib-id can be determined by using the
3076  * '<em>show ip fib</em>' command. If fib-id is not entered, default value
3077  * of 0 is used.
3078  *
3079  * @todo This command uses fib-id, other commands use table-id (not
3080  * just a name, they are different indexes). Would like to change this
3081  * to table-id for consistency.
3082  *
3083  * @cliexpar
3084  * Example of how to run the test lookup command:
3085  * @cliexstart{test lookup 172.16.1.1 table 1 count 2}
3086  * No errors in 2 lookups
3087  * @cliexend
3088 ?*/
3089 /* *INDENT-OFF* */
3090 VLIB_CLI_COMMAND (lookup_test_command, static) = {
3091     .path = "test lookup",
3092     .short_help = "test lookup <ipv4-addr> [table <fib-id>] [count <nn>]",
3093     .function = test_lookup_command_fn,
3094 };
3095 /* *INDENT-ON* */
3096
3097 int vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
3098 {
3099   ip4_main_t * im4 = &ip4_main;
3100   ip4_fib_t * fib;
3101   uword * p = hash_get (im4->fib_index_by_table_id, table_id);
3102
3103   if (p == 0)
3104     return VNET_API_ERROR_NO_SUCH_FIB;
3105
3106   fib = ip4_fib_get (p[0]);
3107
3108   fib->flow_hash_config = flow_hash_config;
3109   return 0;
3110 }
3111
3112 static clib_error_t *
3113 set_ip_flow_hash_command_fn (vlib_main_t * vm,
3114                              unformat_input_t * input,
3115                              vlib_cli_command_t * cmd)
3116 {
3117   int matched = 0;
3118   u32 table_id = 0;
3119   u32 flow_hash_config = 0;
3120   int rv;
3121
3122   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3123     if (unformat (input, "table %d", &table_id))
3124       matched = 1;
3125 #define _(a,v) \
3126     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
3127     foreach_flow_hash_bit
3128 #undef _
3129     else break;
3130   }
3131
3132   if (matched == 0)
3133     return clib_error_return (0, "unknown input `%U'",
3134                               format_unformat_error, input);
3135
3136   rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
3137   switch (rv)
3138     {
3139     case 0:
3140       break;
3141
3142     case VNET_API_ERROR_NO_SUCH_FIB:
3143       return clib_error_return (0, "no such FIB table %d", table_id);
3144
3145     default:
3146       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3147       break;
3148     }
3149
3150   return 0;
3151 }
3152
3153 /*?
3154  * Configure the set of IPv4 fields used by the flow hash.
3155  *
3156  * @cliexpar
3157  * Example of how to set the flow hash on a given table:
3158  * @cliexcmd{set ip flow-hash table 7 dst sport dport proto}
3159  * Example of display the configured flow hash:
3160  * @cliexstart{show ip fib}
3161  * ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
3162  * 0.0.0.0/0
3163  *   unicast-ip4-chain
3164  *   [@0]: dpo-load-balance: [index:0 buckets:1 uRPF:0 to:[0:0]]
3165  *     [0] [@0]: dpo-drop ip6
3166  * 0.0.0.0/32
3167  *   unicast-ip4-chain
3168  *   [@0]: dpo-load-balance: [index:1 buckets:1 uRPF:1 to:[0:0]]
3169  *     [0] [@0]: dpo-drop ip6
3170  * 224.0.0.0/8
3171  *   unicast-ip4-chain
3172  *   [@0]: dpo-load-balance: [index:3 buckets:1 uRPF:3 to:[0:0]]
3173  *     [0] [@0]: dpo-drop ip6
3174  * 6.0.1.2/32
3175  *   unicast-ip4-chain
3176  *   [@0]: dpo-load-balance: [index:30 buckets:1 uRPF:29 to:[0:0]]
3177  *     [0] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
3178  * 7.0.0.1/32
3179  *   unicast-ip4-chain
3180  *   [@0]: dpo-load-balance: [index:31 buckets:4 uRPF:30 to:[0:0]]
3181  *     [0] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3182  *     [1] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3183  *     [2] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3184  *     [3] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
3185  * 240.0.0.0/8
3186  *   unicast-ip4-chain
3187  *   [@0]: dpo-load-balance: [index:2 buckets:1 uRPF:2 to:[0:0]]
3188  *     [0] [@0]: dpo-drop ip6
3189  * 255.255.255.255/32
3190  *   unicast-ip4-chain
3191  *   [@0]: dpo-load-balance: [index:4 buckets:1 uRPF:4 to:[0:0]]
3192  *     [0] [@0]: dpo-drop ip6
3193  * ipv4-VRF:7, fib_index 1, flow hash: dst sport dport proto
3194  * 0.0.0.0/0
3195  *   unicast-ip4-chain
3196  *   [@0]: dpo-load-balance: [index:12 buckets:1 uRPF:11 to:[0:0]]
3197  *     [0] [@0]: dpo-drop ip6
3198  * 0.0.0.0/32
3199  *   unicast-ip4-chain
3200  *   [@0]: dpo-load-balance: [index:13 buckets:1 uRPF:12 to:[0:0]]
3201  *     [0] [@0]: dpo-drop ip6
3202  * 172.16.1.0/24
3203  *   unicast-ip4-chain
3204  *   [@0]: dpo-load-balance: [index:17 buckets:1 uRPF:16 to:[0:0]]
3205  *     [0] [@4]: ipv4-glean: af_packet0
3206  * 172.16.1.1/32
3207  *   unicast-ip4-chain
3208  *   [@0]: dpo-load-balance: [index:18 buckets:1 uRPF:17 to:[1:84]]
3209  *     [0] [@2]: dpo-receive: 172.16.1.1 on af_packet0
3210  * 172.16.1.2/32
3211  *   unicast-ip4-chain
3212  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
3213  *     [0] [@5]: ipv4 via 172.16.1.2 af_packet0: IP4: 02:fe:9e:70:7a:2b -> 26:a5:f6:9c:3a:36
3214  * 172.16.2.0/24
3215  *   unicast-ip4-chain
3216  *   [@0]: dpo-load-balance: [index:19 buckets:1 uRPF:18 to:[0:0]]
3217  *     [0] [@4]: ipv4-glean: af_packet1
3218  * 172.16.2.1/32
3219  *   unicast-ip4-chain
3220  *   [@0]: dpo-load-balance: [index:20 buckets:1 uRPF:19 to:[0:0]]
3221  *     [0] [@2]: dpo-receive: 172.16.2.1 on af_packet1
3222  * 224.0.0.0/8
3223  *   unicast-ip4-chain
3224  *   [@0]: dpo-load-balance: [index:15 buckets:1 uRPF:14 to:[0:0]]
3225  *     [0] [@0]: dpo-drop ip6
3226  * 240.0.0.0/8
3227  *   unicast-ip4-chain
3228  *   [@0]: dpo-load-balance: [index:14 buckets:1 uRPF:13 to:[0:0]]
3229  *     [0] [@0]: dpo-drop ip6
3230  * 255.255.255.255/32
3231  *   unicast-ip4-chain
3232  *   [@0]: dpo-load-balance: [index:16 buckets:1 uRPF:15 to:[0:0]]
3233  *     [0] [@0]: dpo-drop ip6
3234  * @cliexend
3235 ?*/
3236 /* *INDENT-OFF* */
3237 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) = {
3238   .path = "set ip flow-hash",
3239   .short_help =
3240   "set ip flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
3241   .function = set_ip_flow_hash_command_fn,
3242 };
3243 /* *INDENT-ON* */
3244
3245 int vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3246                                  u32 table_index)
3247 {
3248   vnet_main_t * vnm = vnet_get_main();
3249   vnet_interface_main_t * im = &vnm->interface_main;
3250   ip4_main_t * ipm = &ip4_main;
3251   ip_lookup_main_t * lm = &ipm->lookup_main;
3252   vnet_classify_main_t * cm = &vnet_classify_main;
3253   ip4_address_t *if_addr;
3254
3255   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3256     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3257
3258   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3259     return VNET_API_ERROR_NO_SUCH_ENTRY;
3260
3261   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3262   lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
3263
3264   if_addr = ip4_interface_first_address (ipm, sw_if_index, NULL);
3265
3266   if (NULL != if_addr)
3267   {
3268       fib_prefix_t pfx = {
3269           .fp_len = 32,
3270           .fp_proto = FIB_PROTOCOL_IP4,
3271           .fp_addr.ip4 = *if_addr,
3272       };
3273       u32 fib_index;
3274
3275       fib_index = fib_table_get_index_for_sw_if_index(FIB_PROTOCOL_IP4,
3276                                                       sw_if_index);
3277
3278
3279       if (table_index != (u32) ~0)
3280       {
3281           dpo_id_t dpo = DPO_INVALID;
3282
3283           dpo_set(&dpo,
3284                   DPO_CLASSIFY,
3285                   DPO_PROTO_IP4,
3286                   classify_dpo_create(DPO_PROTO_IP4, table_index));
3287
3288           fib_table_entry_special_dpo_add(fib_index,
3289                                           &pfx,
3290                                           FIB_SOURCE_CLASSIFY,
3291                                           FIB_ENTRY_FLAG_NONE,
3292                                           &dpo);
3293           dpo_reset(&dpo);
3294       }
3295       else
3296       {
3297           fib_table_entry_special_remove(fib_index,
3298                                          &pfx,
3299                                          FIB_SOURCE_CLASSIFY);
3300       }
3301   }
3302
3303   return 0;
3304 }
3305
3306 static clib_error_t *
3307 set_ip_classify_command_fn (vlib_main_t * vm,
3308                             unformat_input_t * input,
3309                             vlib_cli_command_t * cmd)
3310 {
3311   u32 table_index = ~0;
3312   int table_index_set = 0;
3313   u32 sw_if_index = ~0;
3314   int rv;
3315
3316   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3317     if (unformat (input, "table-index %d", &table_index))
3318       table_index_set = 1;
3319     else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3320                        vnet_get_main(), &sw_if_index))
3321       ;
3322     else
3323       break;
3324   }
3325
3326   if (table_index_set == 0)
3327     return clib_error_return (0, "classify table-index must be specified");
3328
3329   if (sw_if_index == ~0)
3330     return clib_error_return (0, "interface / subif must be specified");
3331
3332   rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
3333
3334   switch (rv)
3335     {
3336     case 0:
3337       break;
3338
3339     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3340       return clib_error_return (0, "No such interface");
3341
3342     case VNET_API_ERROR_NO_SUCH_ENTRY:
3343       return clib_error_return (0, "No such classifier table");
3344     }
3345   return 0;
3346 }
3347
3348 /*?
3349  * Assign a classification table to an interface. The classification
3350  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3351  * commands. Once the table is create, use this command to filter packets
3352  * on an interface.
3353  *
3354  * @cliexpar
3355  * Example of how to assign a classification table to an interface:
3356  * @cliexcmd{set ip classify intfc GigabitEthernet2/0/0 table-index 1}
3357 ?*/
3358 /* *INDENT-OFF* */
3359 VLIB_CLI_COMMAND (set_ip_classify_command, static) = {
3360     .path = "set ip classify",
3361     .short_help =
3362     "set ip classify intfc <interface> table-index <classify-idx>",
3363     .function = set_ip_classify_command_fn,
3364 };
3365 /* *INDENT-ON* */
3366