src/vnet/ip/ip4_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip4_forward.c: IP v4 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  43 #include <vnet/ethernet/arp_packet.h>   /* for ethernet_arp_header_t */
  44 #include <vnet/ppp/ppp.h>
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vnet/api_errno.h>     /* for API error numbers */
  47 #include <vnet/fib/fib_table.h> /* for FIB table and entry creation */
  48 #include <vnet/fib/fib_entry.h> /* for FIB table and entry creation */
  49 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  50 #include <vnet/fib/ip4_fib.h>
  51 #include <vnet/dpo/load_balance.h>
  52 #include <vnet/dpo/load_balance_map.h>
  53 #include <vnet/dpo/classify_dpo.h>
  54 #include <vnet/mfib/mfib_table.h>       /* for mFIB table and entry creation */
  55
  56 #include <vnet/ip/ip4_forward.h>
  57
  58 /** @brief IPv4 lookup node.
  59     @node ip4-lookup
  60
  61     This is the main IPv4 lookup dispatch node.
  62
  63     @param vm vlib_main_t corresponding to the current thread
  64     @param node vlib_node_runtime_t
  65     @param frame vlib_frame_t whose contents should be dispatched
  66
  67     @par Graph mechanics: buffer metadata, next index usage
  68
  69     @em Uses:
  70     - <code>vnet_buffer(b)->sw_if_index[VLIB_RX]</code>
  71         - Indicates the @c sw_if_index value of the interface that the
  72           packet was received on.
  73     - <code>vnet_buffer(b)->sw_if_index[VLIB_TX]</code>
  74         - When the value is @c ~0 then the node performs a longest prefix
  75           match (LPM) for the packet destination address in the FIB attached
  76           to the receive interface.
  77         - Otherwise perform LPM for the packet destination address in the
  78           indicated FIB. In this case <code>[VLIB_TX]</code> is a FIB index
  79           value (0, 1, ...) and not a VRF id.
  80
  81     @em Sets:
  82     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
  83         - The lookup result adjacency index.
  84
  85     <em>Next Index:</em>
  86     - Dispatches the packet to the node index found in
  87       ip_adjacency_t @c adj->lookup_next_index
  88       (where @c adj is the lookup result adjacency).
  89 */
  90 static uword
  91 ip4_lookup (vlib_main_t * vm,
  92             vlib_node_runtime_t * node, vlib_frame_t * frame)
  93 {
  94   return ip4_lookup_inline (vm, node, frame,
  95                             /* lookup_for_responses_to_locally_received_packets */
  96                             0);
  97
  98 }
  99
 100 static u8 *format_ip4_lookup_trace (u8 * s, va_list * args);
 101
 102 /* *INDENT-OFF* */
 103 VLIB_REGISTER_NODE (ip4_lookup_node) =
 104 {
 105   .function = ip4_lookup,
 106   .name = "ip4-lookup",
 107   .vector_size = sizeof (u32),
 108   .format_trace = format_ip4_lookup_trace,
 109   .n_next_nodes = IP_LOOKUP_N_NEXT,
 110   .next_nodes = IP4_LOOKUP_NEXT_NODES,
 111 };
 112 /* *INDENT-ON* */
 113
 114 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup);
 115
 116 always_inline uword
 117 ip4_load_balance (vlib_main_t * vm,
 118                   vlib_node_runtime_t * node, vlib_frame_t * frame)
 119 {
 120   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 121   u32 n_left_from, n_left_to_next, *from, *to_next;
 122   ip_lookup_next_t next;
 123   u32 thread_index = vlib_get_thread_index ();
 124
 125   from = vlib_frame_vector_args (frame);
 126   n_left_from = frame->n_vectors;
 127   next = node->cached_next_index;
 128
 129   if (node->flags & VLIB_NODE_FLAG_TRACE)
 130     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 131
 132   while (n_left_from > 0)
 133     {
 134       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 135
 136
 137       while (n_left_from >= 4 && n_left_to_next >= 2)
 138         {
 139           ip_lookup_next_t next0, next1;
 140           const load_balance_t *lb0, *lb1;
 141           vlib_buffer_t *p0, *p1;
 142           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 143           const ip4_header_t *ip0, *ip1;
 144           const dpo_id_t *dpo0, *dpo1;
 145
 146           /* Prefetch next iteration. */
 147           {
 148             vlib_buffer_t *p2, *p3;
 149
 150             p2 = vlib_get_buffer (vm, from[2]);
 151             p3 = vlib_get_buffer (vm, from[3]);
 152
 153             vlib_prefetch_buffer_header (p2, STORE);
 154             vlib_prefetch_buffer_header (p3, STORE);
 155
 156             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 157             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 158           }
 159
 160           pi0 = to_next[0] = from[0];
 161           pi1 = to_next[1] = from[1];
 162
 163           from += 2;
 164           n_left_from -= 2;
 165           to_next += 2;
 166           n_left_to_next -= 2;
 167
 168           p0 = vlib_get_buffer (vm, pi0);
 169           p1 = vlib_get_buffer (vm, pi1);
 170
 171           ip0 = vlib_buffer_get_current (p0);
 172           ip1 = vlib_buffer_get_current (p1);
 173           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 174           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 175
 176           lb0 = load_balance_get (lbi0);
 177           lb1 = load_balance_get (lbi1);
 178
 179           /*
 180            * this node is for via FIBs we can re-use the hash value from the
 181            * to node if present.
 182            * We don't want to use the same hash value at each level in the recursion
 183            * graph as that would lead to polarisation
 184            */
 185           hc0 = hc1 = 0;
 186
 187           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 188             {
 189               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 190                 {
 191                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 192                     vnet_buffer (p0)->ip.flow_hash >> 1;
 193                 }
 194               else
 195                 {
 196                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 197                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 198                 }
 199               dpo0 = load_balance_get_fwd_bucket
 200                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 201             }
 202           else
 203             {
 204               dpo0 = load_balance_get_bucket_i (lb0, 0);
 205             }
 206           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 207             {
 208               if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash))
 209                 {
 210                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 211                     vnet_buffer (p1)->ip.flow_hash >> 1;
 212                 }
 213               else
 214                 {
 215                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 216                     ip4_compute_flow_hash (ip1, lb1->lb_hash_config);
 217                 }
 218               dpo1 = load_balance_get_fwd_bucket
 219                 (lb1, (hc1 & (lb1->lb_n_buckets_minus_1)));
 220             }
 221           else
 222             {
 223               dpo1 = load_balance_get_bucket_i (lb1, 0);
 224             }
 225
 226           next0 = dpo0->dpoi_next_node;
 227           next1 = dpo1->dpoi_next_node;
 228
 229           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 230           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 231
 232           vlib_increment_combined_counter
 233             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 234           vlib_increment_combined_counter
 235             (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 236
 237           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 238                                            to_next, n_left_to_next,
 239                                            pi0, pi1, next0, next1);
 240         }
 241
 242       while (n_left_from > 0 && n_left_to_next > 0)
 243         {
 244           ip_lookup_next_t next0;
 245           const load_balance_t *lb0;
 246           vlib_buffer_t *p0;
 247           u32 pi0, lbi0, hc0;
 248           const ip4_header_t *ip0;
 249           const dpo_id_t *dpo0;
 250
 251           pi0 = from[0];
 252           to_next[0] = pi0;
 253           from += 1;
 254           to_next += 1;
 255           n_left_to_next -= 1;
 256           n_left_from -= 1;
 257
 258           p0 = vlib_get_buffer (vm, pi0);
 259
 260           ip0 = vlib_buffer_get_current (p0);
 261           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 262
 263           lb0 = load_balance_get (lbi0);
 264
 265           hc0 = 0;
 266           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 267             {
 268               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 269                 {
 270                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 271                     vnet_buffer (p0)->ip.flow_hash >> 1;
 272                 }
 273               else
 274                 {
 275                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 276                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 277                 }
 278               dpo0 = load_balance_get_fwd_bucket
 279                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 280             }
 281           else
 282             {
 283               dpo0 = load_balance_get_bucket_i (lb0, 0);
 284             }
 285
 286           next0 = dpo0->dpoi_next_node;
 287           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 288
 289           vlib_increment_combined_counter
 290             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 291
 292           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 293                                            to_next, n_left_to_next,
 294                                            pi0, next0);
 295         }
 296
 297       vlib_put_next_frame (vm, node, next, n_left_to_next);
 298     }
 299
 300   return frame->n_vectors;
 301 }
 302
 303 /* *INDENT-OFF* */
 304 VLIB_REGISTER_NODE (ip4_load_balance_node) =
 305 {
 306   .function = ip4_load_balance,
 307   .name = "ip4-load-balance",
 308   .vector_size = sizeof (u32),
 309   .sibling_of = "ip4-lookup",
 310   .format_trace =
 311   format_ip4_lookup_trace,
 312 };
 313 /* *INDENT-ON* */
 314
 315 VLIB_NODE_FUNCTION_MULTIARCH (ip4_load_balance_node, ip4_load_balance);
 316
 317 /* get first interface address */
 318 ip4_address_t *
 319 ip4_interface_first_address (ip4_main_t * im, u32 sw_if_index,
 320                              ip_interface_address_t ** result_ia)
 321 {
 322   ip_lookup_main_t *lm = &im->lookup_main;
 323   ip_interface_address_t *ia = 0;
 324   ip4_address_t *result = 0;
 325
 326   /* *INDENT-OFF* */
 327   foreach_ip_interface_address
 328     (lm, ia, sw_if_index,
 329      1 /* honor unnumbered */ ,
 330      ({
 331        ip4_address_t * a =
 332          ip_interface_address_get_address (lm, ia);
 333        result = a;
 334        break;
 335      }));
 336   /* *INDENT-OFF* */
 337   if (result_ia)
 338     *result_ia = result ? ia : 0;
 339   return result;
 340 }
 341
 342 static void
 343 ip4_add_interface_routes (u32 sw_if_index,
 344                           ip4_main_t * im, u32 fib_index,
 345                           ip_interface_address_t * a)
 346 {
 347   ip_lookup_main_t *lm = &im->lookup_main;
 348   ip4_address_t *address = ip_interface_address_get_address (lm, a);
 349   fib_prefix_t pfx = {
 350     .fp_len = a->address_length,
 351     .fp_proto = FIB_PROTOCOL_IP4,
 352     .fp_addr.ip4 = *address,
 353   };
 354
 355   if (pfx.fp_len <= 30)
 356     {
 357       /* a /30 or shorter - add a glean for the network address */
 358       fib_table_entry_update_one_path (fib_index, &pfx,
 359                                        FIB_SOURCE_INTERFACE,
 360                                        (FIB_ENTRY_FLAG_CONNECTED |
 361                                         FIB_ENTRY_FLAG_ATTACHED),
 362                                        DPO_PROTO_IP4,
 363                                        /* No next-hop address */
 364                                        NULL,
 365                                        sw_if_index,
 366                                        // invalid FIB index
 367                                        ~0,
 368                                        1,
 369                                        // no out-label stack
 370                                        NULL,
 371                                        FIB_ROUTE_PATH_FLAG_NONE);
 372
 373       /* Add the two broadcast addresses as drop */
 374       fib_prefix_t net_pfx = {
 375         .fp_len = 32,
 376         .fp_proto = FIB_PROTOCOL_IP4,
 377         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 378       };
 379       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 380         fib_table_entry_special_add(fib_index,
 381                                     &net_pfx,
 382                                     FIB_SOURCE_INTERFACE,
 383                                     (FIB_ENTRY_FLAG_DROP |
 384                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 385       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 386       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 387         fib_table_entry_special_add(fib_index,
 388                                     &net_pfx,
 389                                     FIB_SOURCE_INTERFACE,
 390                                     (FIB_ENTRY_FLAG_DROP |
 391                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 392     }
 393   else if (pfx.fp_len == 31)
 394     {
 395       u32 mask = clib_host_to_net_u32(1);
 396       fib_prefix_t net_pfx = pfx;
 397
 398       net_pfx.fp_len = 32;
 399       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 400
 401       /* a /31 - add the other end as an attached host */
 402       fib_table_entry_update_one_path (fib_index, &net_pfx,
 403                                        FIB_SOURCE_INTERFACE,
 404                                        (FIB_ENTRY_FLAG_ATTACHED),
 405                                        DPO_PROTO_IP4,
 406                                        &net_pfx.fp_addr,
 407                                        sw_if_index,
 408                                        // invalid FIB index
 409                                        ~0,
 410                                        1,
 411                                        NULL,
 412                                        FIB_ROUTE_PATH_FLAG_NONE);
 413     }
 414   pfx.fp_len = 32;
 415
 416   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 417     {
 418       u32 classify_table_index =
 419         lm->classify_table_index_by_sw_if_index[sw_if_index];
 420       if (classify_table_index != (u32) ~ 0)
 421         {
 422           dpo_id_t dpo = DPO_INVALID;
 423
 424           dpo_set (&dpo,
 425                    DPO_CLASSIFY,
 426                    DPO_PROTO_IP4,
 427                    classify_dpo_create (DPO_PROTO_IP4, classify_table_index));
 428
 429           fib_table_entry_special_dpo_add (fib_index,
 430                                            &pfx,
 431                                            FIB_SOURCE_CLASSIFY,
 432                                            FIB_ENTRY_FLAG_NONE, &dpo);
 433           dpo_reset (&dpo);
 434         }
 435     }
 436
 437   fib_table_entry_update_one_path (fib_index, &pfx,
 438                                    FIB_SOURCE_INTERFACE,
 439                                    (FIB_ENTRY_FLAG_CONNECTED |
 440                                     FIB_ENTRY_FLAG_LOCAL),
 441                                    DPO_PROTO_IP4,
 442                                    &pfx.fp_addr,
 443                                    sw_if_index,
 444                                    // invalid FIB index
 445                                    ~0,
 446                                    1, NULL,
 447                                    FIB_ROUTE_PATH_FLAG_NONE);
 448 }
 449
 450 static void
 451 ip4_del_interface_routes (ip4_main_t * im,
 452                           u32 fib_index,
 453                           ip4_address_t * address, u32 address_length)
 454 {
 455   fib_prefix_t pfx = {
 456     .fp_len = address_length,
 457     .fp_proto = FIB_PROTOCOL_IP4,
 458     .fp_addr.ip4 = *address,
 459   };
 460
 461   if (pfx.fp_len <= 30)
 462     {
 463       fib_prefix_t net_pfx = {
 464         .fp_len = 32,
 465         .fp_proto = FIB_PROTOCOL_IP4,
 466         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 467       };
 468       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 469         fib_table_entry_special_remove(fib_index,
 470                                        &net_pfx,
 471                                        FIB_SOURCE_INTERFACE);
 472       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 473       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 474         fib_table_entry_special_remove(fib_index,
 475                                        &net_pfx,
 476                                        FIB_SOURCE_INTERFACE);
 477       fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 478     }
 479     else if (pfx.fp_len == 31)
 480     {
 481       u32 mask = clib_host_to_net_u32(1);
 482       fib_prefix_t net_pfx = pfx;
 483
 484       net_pfx.fp_len = 32;
 485       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 486
 487       fib_table_entry_delete (fib_index, &net_pfx, FIB_SOURCE_INTERFACE);
 488     }
 489
 490   pfx.fp_len = 32;
 491   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 492 }
 493
 494 void
 495 ip4_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 496 {
 497   ip4_main_t *im = &ip4_main;
 498
 499   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 500
 501   /*
 502    * enable/disable only on the 1<->0 transition
 503    */
 504   if (is_enable)
 505     {
 506       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 507         return;
 508     }
 509   else
 510     {
 511       ASSERT (im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 512       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 513         return;
 514     }
 515   vnet_feature_enable_disable ("ip4-unicast", "ip4-not-enabled", sw_if_index,
 516                                !is_enable, 0, 0);
 517
 518
 519   vnet_feature_enable_disable ("ip4-multicast", "ip4-not-enabled",
 520                                sw_if_index, !is_enable, 0, 0);
 521 }
 522
 523 static clib_error_t *
 524 ip4_add_del_interface_address_internal (vlib_main_t * vm,
 525                                         u32 sw_if_index,
 526                                         ip4_address_t * address,
 527                                         u32 address_length, u32 is_del)
 528 {
 529   vnet_main_t *vnm = vnet_get_main ();
 530   ip4_main_t *im = &ip4_main;
 531   ip_lookup_main_t *lm = &im->lookup_main;
 532   clib_error_t *error = 0;
 533   u32 if_address_index, elts_before;
 534   ip4_address_fib_t ip4_af, *addr_fib = 0;
 535
 536   /* local0 interface doesn't support IP addressing  */
 537   if (sw_if_index == 0)
 538     {
 539       return
 540        clib_error_create ("local0 interface doesn't support IP addressing");
 541     }
 542
 543   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 544   ip4_addr_fib_init (&ip4_af, address,
 545                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 546   vec_add1 (addr_fib, ip4_af);
 547
 548   /* FIXME-LATER
 549    * there is no support for adj-fib handling in the presence of overlapping
 550    * subnets on interfaces. Easy fix - disallow overlapping subnets, like
 551    * most routers do.
 552    */
 553   /* *INDENT-OFF* */
 554   if (!is_del)
 555     {
 556       /* When adding an address check that it does not conflict
 557          with an existing address. */
 558       ip_interface_address_t *ia;
 559       foreach_ip_interface_address
 560         (&im->lookup_main, ia, sw_if_index,
 561          0 /* honor unnumbered */ ,
 562          ({
 563            ip4_address_t * x =
 564              ip_interface_address_get_address
 565              (&im->lookup_main, ia);
 566            if (ip4_destination_matches_route
 567                (im, address, x, ia->address_length) ||
 568                ip4_destination_matches_route (im,
 569                                               x,
 570                                               address,
 571                                               address_length))
 572              return
 573                clib_error_create
 574                ("failed to add %U which conflicts with %U for interface %U",
 575                 format_ip4_address_and_length, address,
 576                 address_length,
 577                 format_ip4_address_and_length, x,
 578                 ia->address_length,
 579                 format_vnet_sw_if_index_name, vnm,
 580                 sw_if_index);
 581          }));
 582     }
 583   /* *INDENT-ON* */
 584
 585   elts_before = pool_elts (lm->if_address_pool);
 586
 587   error = ip_interface_address_add_del
 588     (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 589   if (error)
 590     goto done;
 591
 592   ip4_sw_interface_enable_disable (sw_if_index, !is_del);
 593
 594   if (is_del)
 595     ip4_del_interface_routes (im, ip4_af.fib_index, address, address_length);
 596   else
 597     ip4_add_interface_routes (sw_if_index,
 598                               im, ip4_af.fib_index,
 599                               pool_elt_at_index
 600                               (lm->if_address_pool, if_address_index));
 601
 602   /* If pool did not grow/shrink: add duplicate address. */
 603   if (elts_before != pool_elts (lm->if_address_pool))
 604     {
 605       ip4_add_del_interface_address_callback_t *cb;
 606       vec_foreach (cb, im->add_del_interface_address_callbacks)
 607         cb->function (im, cb->function_opaque, sw_if_index,
 608                       address, address_length, if_address_index, is_del);
 609     }
 610
 611 done:
 612   vec_free (addr_fib);
 613   return error;
 614 }
 615
 616 clib_error_t *
 617 ip4_add_del_interface_address (vlib_main_t * vm,
 618                                u32 sw_if_index,
 619                                ip4_address_t * address,
 620                                u32 address_length, u32 is_del)
 621 {
 622   return ip4_add_del_interface_address_internal
 623     (vm, sw_if_index, address, address_length, is_del);
 624 }
 625
 626 /* Built-in ip4 unicast rx feature path definition */
 627 /* *INDENT-OFF* */
 628 VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
 629 {
 630   .arc_name = "ip4-unicast",
 631   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 632   .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
 633 };
 634
 635 VNET_FEATURE_INIT (ip4_flow_classify, static) =
 636 {
 637   .arc_name = "ip4-unicast",
 638   .node_name = "ip4-flow-classify",
 639   .runs_before = VNET_FEATURES ("ip4-inacl"),
 640 };
 641
 642 VNET_FEATURE_INIT (ip4_inacl, static) =
 643 {
 644   .arc_name = "ip4-unicast",
 645   .node_name = "ip4-inacl",
 646   .runs_before = VNET_FEATURES ("ip4-source-check-via-rx"),
 647 };
 648
 649 VNET_FEATURE_INIT (ip4_source_check_1, static) =
 650 {
 651   .arc_name = "ip4-unicast",
 652   .node_name = "ip4-source-check-via-rx",
 653   .runs_before = VNET_FEATURES ("ip4-source-check-via-any"),
 654 };
 655
 656 VNET_FEATURE_INIT (ip4_source_check_2, static) =
 657 {
 658   .arc_name = "ip4-unicast",
 659   .node_name = "ip4-source-check-via-any",
 660   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
 661 };
 662
 663 VNET_FEATURE_INIT (ip4_source_and_port_range_check_rx, static) =
 664 {
 665   .arc_name = "ip4-unicast",
 666   .node_name = "ip4-source-and-port-range-check-rx",
 667   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
 668 };
 669
 670 VNET_FEATURE_INIT (ip4_policer_classify, static) =
 671 {
 672   .arc_name = "ip4-unicast",
 673   .node_name = "ip4-policer-classify",
 674   .runs_before = VNET_FEATURES ("ipsec-input-ip4"),
 675 };
 676
 677 VNET_FEATURE_INIT (ip4_ipsec, static) =
 678 {
 679   .arc_name = "ip4-unicast",
 680   .node_name = "ipsec-input-ip4",
 681   .runs_before = VNET_FEATURES ("vpath-input-ip4"),
 682 };
 683
 684 VNET_FEATURE_INIT (ip4_vpath, static) =
 685 {
 686   .arc_name = "ip4-unicast",
 687   .node_name = "vpath-input-ip4",
 688   .runs_before = VNET_FEATURES ("ip4-vxlan-bypass"),
 689 };
 690
 691 VNET_FEATURE_INIT (ip4_vxlan_bypass, static) =
 692 {
 693   .arc_name = "ip4-unicast",
 694   .node_name = "ip4-vxlan-bypass",
 695   .runs_before = VNET_FEATURES ("ip4-lookup"),
 696 };
 697
 698 VNET_FEATURE_INIT (ip4_not_enabled, static) =
 699 {
 700   .arc_name = "ip4-unicast",
 701   .node_name = "ip4-not-enabled",
 702   .runs_before = VNET_FEATURES ("ip4-lookup"),
 703 };
 704
 705 VNET_FEATURE_INIT (ip4_lookup, static) =
 706 {
 707   .arc_name = "ip4-unicast",
 708   .node_name = "ip4-lookup",
 709   .runs_before = 0,     /* not before any other features */
 710 };
 711
 712 /* Built-in ip4 multicast rx feature path definition */
 713 VNET_FEATURE_ARC_INIT (ip4_multicast, static) =
 714 {
 715   .arc_name = "ip4-multicast",
 716   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 717   .arc_index_ptr = &ip4_main.lookup_main.mcast_feature_arc_index,
 718 };
 719
 720 VNET_FEATURE_INIT (ip4_vpath_mc, static) =
 721 {
 722   .arc_name = "ip4-multicast",
 723   .node_name = "vpath-input-ip4",
 724   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
 725 };
 726
 727 VNET_FEATURE_INIT (ip4_mc_not_enabled, static) =
 728 {
 729   .arc_name = "ip4-multicast",
 730   .node_name = "ip4-not-enabled",
 731   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
 732 };
 733
 734 VNET_FEATURE_INIT (ip4_lookup_mc, static) =
 735 {
 736   .arc_name = "ip4-multicast",
 737   .node_name = "ip4-mfib-forward-lookup",
 738   .runs_before = 0,     /* last feature */
 739 };
 740
 741 /* Source and port-range check ip4 tx feature path definition */
 742 VNET_FEATURE_ARC_INIT (ip4_output, static) =
 743 {
 744   .arc_name = "ip4-output",
 745   .start_nodes = VNET_FEATURES ("ip4-rewrite", "ip4-midchain", "ip4-dvr-dpo"),
 746   .arc_index_ptr = &ip4_main.lookup_main.output_feature_arc_index,
 747 };
 748
 749 VNET_FEATURE_INIT (ip4_source_and_port_range_check_tx, static) =
 750 {
 751   .arc_name = "ip4-output",
 752   .node_name = "ip4-source-and-port-range-check-tx",
 753   .runs_before = VNET_FEATURES ("ip4-outacl"),
 754 };
 755
 756 VNET_FEATURE_INIT (ip4_outacl, static) =
 757 {
 758   .arc_name = "ip4-output",
 759   .node_name = "ip4-outacl",
 760   .runs_before = VNET_FEATURES ("ipsec-output-ip4"),
 761 };
 762
 763 VNET_FEATURE_INIT (ip4_ipsec_output, static) =
 764 {
 765   .arc_name = "ip4-output",
 766   .node_name = "ipsec-output-ip4",
 767   .runs_before = VNET_FEATURES ("interface-output"),
 768 };
 769
 770 /* Built-in ip4 tx feature path definition */
 771 VNET_FEATURE_INIT (ip4_interface_output, static) =
 772 {
 773   .arc_name = "ip4-output",
 774   .node_name = "interface-output",
 775   .runs_before = 0,     /* not before any other features */
 776 };
 777 /* *INDENT-ON* */
 778
 779 static clib_error_t *
 780 ip4_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 781 {
 782   ip4_main_t *im = &ip4_main;
 783
 784   /* Fill in lookup tables with default table (0). */
 785   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 786   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 787
 788   if (!is_add)
 789     {
 790       ip4_main_t *im4 = &ip4_main;
 791       ip_lookup_main_t *lm4 = &im4->lookup_main;
 792       ip_interface_address_t *ia = 0;
 793       ip4_address_t *address;
 794       vlib_main_t *vm = vlib_get_main ();
 795
 796       /* *INDENT-OFF* */
 797       foreach_ip_interface_address (lm4, ia, sw_if_index, 1 /* honor unnumbered */,
 798       ({
 799         address = ip_interface_address_get_address (lm4, ia);
 800         ip4_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
 801       }));
 802       /* *INDENT-ON* */
 803     }
 804
 805   vnet_feature_enable_disable ("ip4-unicast", "ip4-not-enabled", sw_if_index,
 806                                is_add, 0, 0);
 807
 808   vnet_feature_enable_disable ("ip4-multicast", "ip4-not-enabled",
 809                                sw_if_index, is_add, 0, 0);
 810
 811   return /* no error */ 0;
 812 }
 813
 814 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
 815
 816 /* Global IP4 main. */
 817 ip4_main_t ip4_main;
 818
 819 clib_error_t *
 820 ip4_lookup_init (vlib_main_t * vm)
 821 {
 822   ip4_main_t *im = &ip4_main;
 823   clib_error_t *error;
 824   uword i;
 825
 826   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
 827     return error;
 828   if ((error = vlib_call_init_function (vm, ip4_mtrie_module_init)))
 829     return (error);
 830   if ((error = vlib_call_init_function (vm, fib_module_init)))
 831     return error;
 832   if ((error = vlib_call_init_function (vm, mfib_module_init)))
 833     return error;
 834
 835   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
 836     {
 837       u32 m;
 838
 839       if (i < 32)
 840         m = pow2_mask (i) << (32 - i);
 841       else
 842         m = ~0;
 843       im->fib_masks[i] = clib_host_to_net_u32 (m);
 844     }
 845
 846   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
 847
 848   /* Create FIB with index 0 and table id of 0. */
 849   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0,
 850                                      FIB_SOURCE_DEFAULT_ROUTE);
 851   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0,
 852                                       MFIB_SOURCE_DEFAULT_ROUTE);
 853
 854   {
 855     pg_node_t *pn;
 856     pn = pg_get_node (ip4_lookup_node.index);
 857     pn->unformat_edit = unformat_pg_ip4_header;
 858   }
 859
 860   {
 861     ethernet_arp_header_t h;
 862
 863     memset (&h, 0, sizeof (h));
 864
 865     /* Set target ethernet address to all zeros. */
 866     memset (h.ip4_over_ethernet[1].ethernet, 0,
 867             sizeof (h.ip4_over_ethernet[1].ethernet));
 868
 869 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
 870 #define _8(f,v) h.f = v;
 871     _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
 872     _16 (l3_type, ETHERNET_TYPE_IP4);
 873     _8 (n_l2_address_bytes, 6);
 874     _8 (n_l3_address_bytes, 4);
 875     _16 (opcode, ETHERNET_ARP_OPCODE_request);
 876 #undef _16
 877 #undef _8
 878
 879     vlib_packet_template_init (vm, &im->ip4_arp_request_packet_template,
 880                                /* data */ &h,
 881                                sizeof (h),
 882                                /* alloc chunk size */ 8,
 883                                "ip4 arp");
 884   }
 885
 886   return error;
 887 }
 888
 889 VLIB_INIT_FUNCTION (ip4_lookup_init);
 890
 891 typedef struct
 892 {
 893   /* Adjacency taken. */
 894   u32 dpo_index;
 895   u32 flow_hash;
 896   u32 fib_index;
 897
 898   /* Packet data, possibly *after* rewrite. */
 899   u8 packet_data[64 - 1 * sizeof (u32)];
 900 }
 901 ip4_forward_next_trace_t;
 902
 903 u8 *
 904 format_ip4_forward_next_trace (u8 * s, va_list * args)
 905 {
 906   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 907   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 908   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
 909   u32 indent = format_get_indent (s);
 910   s = format (s, "%U%U",
 911               format_white_space, indent,
 912               format_ip4_header, t->packet_data, sizeof (t->packet_data));
 913   return s;
 914 }
 915
 916 static u8 *
 917 format_ip4_lookup_trace (u8 * s, va_list * args)
 918 {
 919   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 920   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 921   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
 922   u32 indent = format_get_indent (s);
 923
 924   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 925               t->fib_index, t->dpo_index, t->flow_hash);
 926   s = format (s, "\n%U%U",
 927               format_white_space, indent,
 928               format_ip4_header, t->packet_data, sizeof (t->packet_data));
 929   return s;
 930 }
 931
 932 static u8 *
 933 format_ip4_rewrite_trace (u8 * s, va_list * args)
 934 {
 935   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 936   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 937   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
 938   u32 indent = format_get_indent (s);
 939
 940   s = format (s, "tx_sw_if_index %d dpo-idx %d : %U flow hash: 0x%08x",
 941               t->fib_index, t->dpo_index, format_ip_adjacency,
 942               t->dpo_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
 943   s = format (s, "\n%U%U",
 944               format_white_space, indent,
 945               format_ip_adjacency_packet_data,
 946               t->dpo_index, t->packet_data, sizeof (t->packet_data));
 947   return s;
 948 }
 949
 950 /* Common trace function for all ip4-forward next nodes. */
 951 void
 952 ip4_forward_next_trace (vlib_main_t * vm,
 953                         vlib_node_runtime_t * node,
 954                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
 955 {
 956   u32 *from, n_left;
 957   ip4_main_t *im = &ip4_main;
 958
 959   n_left = frame->n_vectors;
 960   from = vlib_frame_vector_args (frame);
 961
 962   while (n_left >= 4)
 963     {
 964       u32 bi0, bi1;
 965       vlib_buffer_t *b0, *b1;
 966       ip4_forward_next_trace_t *t0, *t1;
 967
 968       /* Prefetch next iteration. */
 969       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
 970       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
 971
 972       bi0 = from[0];
 973       bi1 = from[1];
 974
 975       b0 = vlib_get_buffer (vm, bi0);
 976       b1 = vlib_get_buffer (vm, bi1);
 977
 978       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 979         {
 980           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 981           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
 982           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
 983           t0->fib_index =
 984             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
 985              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
 986             vec_elt (im->fib_index_by_sw_if_index,
 987                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
 988
 989           clib_memcpy (t0->packet_data,
 990                        vlib_buffer_get_current (b0),
 991                        sizeof (t0->packet_data));
 992         }
 993       if (b1->flags & VLIB_BUFFER_IS_TRACED)
 994         {
 995           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
 996           t1->dpo_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
 997           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
 998           t1->fib_index =
 999             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
1000              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
1001             vec_elt (im->fib_index_by_sw_if_index,
1002                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
1003           clib_memcpy (t1->packet_data, vlib_buffer_get_current (b1),
1004                        sizeof (t1->packet_data));
1005         }
1006       from += 2;
1007       n_left -= 2;
1008     }
1009
1010   while (n_left >= 1)
1011     {
1012       u32 bi0;
1013       vlib_buffer_t *b0;
1014       ip4_forward_next_trace_t *t0;
1015
1016       bi0 = from[0];
1017
1018       b0 = vlib_get_buffer (vm, bi0);
1019
1020       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1021         {
1022           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1023           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1024           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1025           t0->fib_index =
1026             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1027              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1028             vec_elt (im->fib_index_by_sw_if_index,
1029                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1030           clib_memcpy (t0->packet_data, vlib_buffer_get_current (b0),
1031                        sizeof (t0->packet_data));
1032         }
1033       from += 1;
1034       n_left -= 1;
1035     }
1036 }
1037
1038 /* Compute TCP/UDP/ICMP4 checksum in software. */
1039 u16
1040 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1041                               ip4_header_t * ip0)
1042 {
1043   ip_csum_t sum0;
1044   u32 ip_header_length, payload_length_host_byte_order;
1045   u32 n_this_buffer, n_bytes_left, n_ip_bytes_this_buffer;
1046   u16 sum16;
1047   void *data_this_buffer;
1048
1049   /* Initialize checksum with ip header. */
1050   ip_header_length = ip4_header_bytes (ip0);
1051   payload_length_host_byte_order =
1052     clib_net_to_host_u16 (ip0->length) - ip_header_length;
1053   sum0 =
1054     clib_host_to_net_u32 (payload_length_host_byte_order +
1055                           (ip0->protocol << 16));
1056
1057   if (BITS (uword) == 32)
1058     {
1059       sum0 =
1060         ip_csum_with_carry (sum0,
1061                             clib_mem_unaligned (&ip0->src_address, u32));
1062       sum0 =
1063         ip_csum_with_carry (sum0,
1064                             clib_mem_unaligned (&ip0->dst_address, u32));
1065     }
1066   else
1067     sum0 =
1068       ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1069
1070   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1071   data_this_buffer = (void *) ip0 + ip_header_length;
1072   n_ip_bytes_this_buffer =
1073     p0->current_length - (((u8 *) ip0 - p0->data) - p0->current_data);
1074   if (n_this_buffer + ip_header_length > n_ip_bytes_this_buffer)
1075     {
1076       n_this_buffer = n_ip_bytes_this_buffer > ip_header_length ?
1077         n_ip_bytes_this_buffer - ip_header_length : 0;
1078     }
1079   while (1)
1080     {
1081       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1082       n_bytes_left -= n_this_buffer;
1083       if (n_bytes_left == 0)
1084         break;
1085
1086       ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1087       p0 = vlib_get_buffer (vm, p0->next_buffer);
1088       data_this_buffer = vlib_buffer_get_current (p0);
1089       n_this_buffer = p0->current_length;
1090     }
1091
1092   sum16 = ~ip_csum_fold (sum0);
1093
1094   return sum16;
1095 }
1096
1097 u32
1098 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1099 {
1100   ip4_header_t *ip0 = vlib_buffer_get_current (p0);
1101   udp_header_t *udp0;
1102   u16 sum16;
1103
1104   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1105           || ip0->protocol == IP_PROTOCOL_UDP);
1106
1107   udp0 = (void *) (ip0 + 1);
1108   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1109     {
1110       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1111                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
1112       return p0->flags;
1113     }
1114
1115   sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1116
1117   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1118                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
1119
1120   return p0->flags;
1121 }
1122
1123 /* *INDENT-OFF* */
1124 VNET_FEATURE_ARC_INIT (ip4_local) =
1125 {
1126   .arc_name  = "ip4-local",
1127   .start_nodes = VNET_FEATURES ("ip4-local"),
1128 };
1129 /* *INDENT-ON* */
1130
1131 static inline void
1132 ip4_local_validate_l4 (vlib_main_t * vm, vlib_buffer_t * p, ip4_header_t * ip,
1133                        u8 is_udp, u8 * error, u8 * good_tcp_udp)
1134 {
1135   u32 flags0;
1136   flags0 = ip4_tcp_udp_validate_checksum (vm, p);
1137   *good_tcp_udp = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1138   if (is_udp)
1139     {
1140       udp_header_t *udp;
1141       u32 ip_len, udp_len;
1142       i32 len_diff;
1143       udp = ip4_next_header (ip);
1144       /* Verify UDP length. */
1145       ip_len = clib_net_to_host_u16 (ip->length);
1146       udp_len = clib_net_to_host_u16 (udp->length);
1147
1148       len_diff = ip_len - udp_len;
1149       *good_tcp_udp &= len_diff >= 0;
1150       *error = len_diff < 0 ? IP4_ERROR_UDP_LENGTH : *error;
1151     }
1152 }
1153
1154 #define ip4_local_do_l4_check(is_tcp_udp, flags)                        \
1155     (is_tcp_udp && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED \
1156     || flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM \
1157     || flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
1158
1159 static inline uword
1160 ip4_local_inline (vlib_main_t * vm,
1161                   vlib_node_runtime_t * node,
1162                   vlib_frame_t * frame, int head_of_feature_arc)
1163 {
1164   ip4_main_t *im = &ip4_main;
1165   ip_lookup_main_t *lm = &im->lookup_main;
1166   ip_local_next_t next_index;
1167   u32 *from, *to_next, n_left_from, n_left_to_next;
1168   vlib_node_runtime_t *error_node =
1169     vlib_node_get_runtime (vm, ip4_input_node.index);
1170   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1171
1172   from = vlib_frame_vector_args (frame);
1173   n_left_from = frame->n_vectors;
1174   next_index = node->cached_next_index;
1175
1176   if (node->flags & VLIB_NODE_FLAG_TRACE)
1177     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1178
1179   while (n_left_from > 0)
1180     {
1181       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1182
1183       while (n_left_from >= 4 && n_left_to_next >= 2)
1184         {
1185           vlib_buffer_t *p0, *p1;
1186           ip4_header_t *ip0, *ip1;
1187           ip4_fib_mtrie_t *mtrie0, *mtrie1;
1188           ip4_fib_mtrie_leaf_t leaf0, leaf1;
1189           const dpo_id_t *dpo0, *dpo1;
1190           const load_balance_t *lb0, *lb1;
1191           u32 pi0, next0, fib_index0, lbi0;
1192           u32 pi1, next1, fib_index1, lbi1;
1193           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1194           u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1195           u32 sw_if_index0, sw_if_index1;
1196
1197           pi0 = to_next[0] = from[0];
1198           pi1 = to_next[1] = from[1];
1199           from += 2;
1200           n_left_from -= 2;
1201           to_next += 2;
1202           n_left_to_next -= 2;
1203
1204           next0 = next1 = IP_LOCAL_NEXT_DROP;
1205           error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1206
1207           p0 = vlib_get_buffer (vm, pi0);
1208           p1 = vlib_get_buffer (vm, pi1);
1209
1210           ip0 = vlib_buffer_get_current (p0);
1211           ip1 = vlib_buffer_get_current (p1);
1212
1213           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1214           vnet_buffer (p1)->l3_hdr_offset = p1->current_data;
1215
1216           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1217           sw_if_index1 = vnet_buffer (p1)->sw_if_index[VLIB_RX];
1218
1219           /* Treat IP frag packets as "experimental" protocol for now
1220              until support of IP frag reassembly is implemented */
1221           proto0 =
1222             ip4_is_fragment (ip0) ? IP_PROTOCOL_VPP_FRAGMENTATION :
1223             ip0->protocol;
1224           proto1 =
1225             ip4_is_fragment (ip1) ? IP_PROTOCOL_VPP_FRAGMENTATION :
1226             ip1->protocol;
1227
1228           if (head_of_feature_arc == 0)
1229             goto skip_checks;
1230
1231           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1232           is_udp1 = proto1 == IP_PROTOCOL_UDP;
1233           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1234           is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1235
1236           good_tcp_udp0 =
1237             (p0->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1238              || (p0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1239                  || p0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) != 0;
1240           good_tcp_udp1 = (p1->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1241                            || (p1->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1242                                || p1->flags &
1243                                VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) != 0;
1244
1245           if (PREDICT_FALSE (ip4_local_do_l4_check (is_tcp_udp0, p0->flags)
1246                              || ip4_local_do_l4_check (is_tcp_udp1,
1247                                                        p1->flags)))
1248             {
1249               if (is_tcp_udp0)
1250                 ip4_local_validate_l4 (vm, p0, ip0, is_udp0, &error0,
1251                                        &good_tcp_udp0);
1252               if (is_tcp_udp1)
1253                 ip4_local_validate_l4 (vm, p1, ip1, is_udp1, &error1,
1254                                        &good_tcp_udp1);
1255             }
1256
1257           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1258           error0 = (is_tcp_udp0 && !good_tcp_udp0
1259                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1260           error1 = (is_tcp_udp1 && !good_tcp_udp1
1261                     ? IP4_ERROR_TCP_CHECKSUM + is_udp1 : error1);
1262
1263           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1264           fib_index0 =
1265             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1266              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1267
1268           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index1);
1269           fib_index1 =
1270             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
1271              (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
1272
1273           /* TODO maybe move to lookup? */
1274           vnet_buffer (p0)->ip.fib_index = fib_index0;
1275           vnet_buffer (p1)->ip.fib_index = fib_index1;
1276
1277           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1278           mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
1279
1280           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1281           leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, &ip1->src_address);
1282           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1283                                              2);
1284           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address,
1285                                              2);
1286           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1287                                              3);
1288           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address,
1289                                              3);
1290
1291           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0 =
1292             ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1293           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1294
1295           vnet_buffer (p1)->ip.adj_index[VLIB_RX] = lbi1 =
1296             ip4_fib_mtrie_leaf_get_adj_index (leaf1);
1297           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = lbi1;
1298
1299           lb0 = load_balance_get (lbi0);
1300           lb1 = load_balance_get (lbi1);
1301           dpo0 = load_balance_get_bucket_i (lb0, 0);
1302           dpo1 = load_balance_get_bucket_i (lb1, 0);
1303
1304           /*
1305            * Must have a route to source otherwise we drop the packet.
1306            * ip4 broadcasts are accepted, e.g. to make dhcp client work
1307            *
1308            * The checks are:
1309            *  - the source is a recieve => it's from us => bogus, do this
1310            *    first since it sets a different error code.
1311            *  - uRPF check for any route to source - accept if passes.
1312            *  - allow packets destined to the broadcast address from unknown sources
1313            */
1314           if (p0->flags & VNET_BUFFER_F_IS_NATED)
1315             goto skip_check0;
1316
1317           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1318                      dpo0->dpoi_type == DPO_RECEIVE) ?
1319                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1320           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1321                      !fib_urpf_check_size (lb0->lb_urpf) &&
1322                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1323                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1324
1325         skip_check0:
1326           if (p1->flags & VNET_BUFFER_F_IS_NATED)
1327             goto skip_checks;
1328
1329           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1330                      dpo1->dpoi_type == DPO_RECEIVE) ?
1331                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error1);
1332           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1333                      !fib_urpf_check_size (lb1->lb_urpf) &&
1334                      ip1->dst_address.as_u32 != 0xFFFFFFFF)
1335                     ? IP4_ERROR_SRC_LOOKUP_MISS : error1);
1336
1337         skip_checks:
1338
1339           next0 = lm->local_next_by_ip_protocol[proto0];
1340           next1 = lm->local_next_by_ip_protocol[proto1];
1341
1342           next0 =
1343             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1344           next1 =
1345             error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1346
1347           p0->error = error0 ? error_node->errors[error0] : 0;
1348           p1->error = error1 ? error_node->errors[error1] : 0;
1349
1350           if (head_of_feature_arc)
1351             {
1352               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1353                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1354               if (PREDICT_TRUE (error1 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1355                 vnet_feature_arc_start (arc_index, sw_if_index1, &next1, p1);
1356             }
1357
1358           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1359                                            n_left_to_next, pi0, pi1,
1360                                            next0, next1);
1361         }
1362
1363       while (n_left_from > 0 && n_left_to_next > 0)
1364         {
1365           vlib_buffer_t *p0;
1366           ip4_header_t *ip0;
1367           ip4_fib_mtrie_t *mtrie0;
1368           ip4_fib_mtrie_leaf_t leaf0;
1369           u32 pi0, next0, fib_index0, lbi0;
1370           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1371           load_balance_t *lb0;
1372           const dpo_id_t *dpo0;
1373           u32 sw_if_index0;
1374
1375           pi0 = to_next[0] = from[0];
1376           from += 1;
1377           n_left_from -= 1;
1378           to_next += 1;
1379           n_left_to_next -= 1;
1380
1381           next0 = IP_LOCAL_NEXT_DROP;
1382           error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
1383
1384           p0 = vlib_get_buffer (vm, pi0);
1385           ip0 = vlib_buffer_get_current (p0);
1386           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1387           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1388
1389           /* Treat IP frag packets as "experimental" protocol for now
1390              until support of IP frag reassembly is implemented */
1391           proto0 =
1392             ip4_is_fragment (ip0) ? IP_PROTOCOL_VPP_FRAGMENTATION :
1393             ip0->protocol;
1394
1395           if (head_of_feature_arc == 0 || p0->flags & VNET_BUFFER_F_IS_NATED)
1396             goto skip_check;
1397
1398           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1399           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1400
1401           good_tcp_udp0 =
1402             (p0->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1403              || (p0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1404                  || p0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) != 0;
1405
1406           if (PREDICT_FALSE (ip4_local_do_l4_check (is_tcp_udp0, p0->flags)))
1407             {
1408               ip4_local_validate_l4 (vm, p0, ip0, is_udp0, &error0,
1409                                      &good_tcp_udp0);
1410             }
1411
1412           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1413           error0 = (is_tcp_udp0 && !good_tcp_udp0
1414                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1415
1416           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1417           fib_index0 =
1418             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1419              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1420           vnet_buffer (p0)->ip.fib_index = fib_index0;
1421           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1422           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1423           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1424                                              2);
1425           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1426                                              3);
1427           lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1428           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1429           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0;
1430
1431           lb0 = load_balance_get (lbi0);
1432           dpo0 = load_balance_get_bucket_i (lb0, 0);
1433
1434           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1435                      dpo0->dpoi_type == DPO_RECEIVE) ?
1436                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1437           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1438                      !fib_urpf_check_size (lb0->lb_urpf) &&
1439                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1440                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1441
1442         skip_check:
1443           next0 = lm->local_next_by_ip_protocol[proto0];
1444           next0 =
1445             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1446
1447           p0->error = error0 ? error_node->errors[error0] : 0;
1448
1449           if (head_of_feature_arc)
1450             {
1451               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1452                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1453             }
1454
1455           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1456                                            n_left_to_next, pi0, next0);
1457         }
1458       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1459     }
1460
1461   return frame->n_vectors;
1462 }
1463
1464 static uword
1465 ip4_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1466 {
1467   return ip4_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1468 }
1469
1470 /* *INDENT-OFF* */
1471 VLIB_REGISTER_NODE (ip4_local_node) =
1472 {
1473   .function = ip4_local,
1474   .name = "ip4-local",
1475   .vector_size = sizeof (u32),
1476   .format_trace = format_ip4_forward_next_trace,
1477   .n_next_nodes = IP_LOCAL_N_NEXT,
1478   .next_nodes =
1479   {
1480     [IP_LOCAL_NEXT_DROP] = "ip4-drop",
1481     [IP_LOCAL_NEXT_PUNT] = "ip4-punt",
1482     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
1483     [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",
1484     [IP_LOCAL_NEXT_REASSEMBLY] = "ip4-reassembly",
1485   },
1486 };
1487 /* *INDENT-ON* */
1488
1489 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local);
1490
1491 static uword
1492 ip4_local_end_of_arc (vlib_main_t * vm,
1493                       vlib_node_runtime_t * node, vlib_frame_t * frame)
1494 {
1495   return ip4_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1496 }
1497
1498 /* *INDENT-OFF* */
1499 VLIB_REGISTER_NODE (ip4_local_end_of_arc_node,static) = {
1500   .function = ip4_local_end_of_arc,
1501   .name = "ip4-local-end-of-arc",
1502   .vector_size = sizeof (u32),
1503
1504   .format_trace = format_ip4_forward_next_trace,
1505   .sibling_of = "ip4-local",
1506 };
1507
1508 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_end_of_arc_node, ip4_local_end_of_arc)
1509
1510 VNET_FEATURE_INIT (ip4_local_end_of_arc, static) = {
1511   .arc_name = "ip4-local",
1512   .node_name = "ip4-local-end-of-arc",
1513   .runs_before = 0, /* not before any other features */
1514 };
1515 /* *INDENT-ON* */
1516
1517 void
1518 ip4_register_protocol (u32 protocol, u32 node_index)
1519 {
1520   vlib_main_t *vm = vlib_get_main ();
1521   ip4_main_t *im = &ip4_main;
1522   ip_lookup_main_t *lm = &im->lookup_main;
1523
1524   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1525   lm->local_next_by_ip_protocol[protocol] =
1526     vlib_node_add_next (vm, ip4_local_node.index, node_index);
1527 }
1528
1529 static clib_error_t *
1530 show_ip_local_command_fn (vlib_main_t * vm,
1531                           unformat_input_t * input, vlib_cli_command_t * cmd)
1532 {
1533   ip4_main_t *im = &ip4_main;
1534   ip_lookup_main_t *lm = &im->lookup_main;
1535   int i;
1536
1537   vlib_cli_output (vm, "Protocols handled by ip4_local");
1538   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
1539     {
1540       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
1541         {
1542           u32 node_index = vlib_get_node (vm,
1543                                           ip4_local_node.index)->
1544             next_nodes[lm->local_next_by_ip_protocol[i]];
1545           vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm,
1546                            node_index);
1547         }
1548     }
1549   return 0;
1550 }
1551
1552
1553
1554 /*?
1555  * Display the set of protocols handled by the local IPv4 stack.
1556  *
1557  * @cliexpar
1558  * Example of how to display local protocol table:
1559  * @cliexstart{show ip local}
1560  * Protocols handled by ip4_local
1561  * 1
1562  * 17
1563  * 47
1564  * @cliexend
1565 ?*/
1566 /* *INDENT-OFF* */
1567 VLIB_CLI_COMMAND (show_ip_local, static) =
1568 {
1569   .path = "show ip local",
1570   .function = show_ip_local_command_fn,
1571   .short_help = "show ip local",
1572 };
1573 /* *INDENT-ON* */
1574
1575 always_inline uword
1576 ip4_arp_inline (vlib_main_t * vm,
1577                 vlib_node_runtime_t * node,
1578                 vlib_frame_t * frame, int is_glean)
1579 {
1580   vnet_main_t *vnm = vnet_get_main ();
1581   ip4_main_t *im = &ip4_main;
1582   ip_lookup_main_t *lm = &im->lookup_main;
1583   u32 *from, *to_next_drop;
1584   uword n_left_from, n_left_to_next_drop, next_index;
1585   static f64 time_last_seed_change = -1e100;
1586   static u32 hash_seeds[3];
1587   static uword hash_bitmap[256 / BITS (uword)];
1588   f64 time_now;
1589
1590   if (node->flags & VLIB_NODE_FLAG_TRACE)
1591     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1592
1593   time_now = vlib_time_now (vm);
1594   if (time_now - time_last_seed_change > 1e-3)
1595     {
1596       uword i;
1597       u32 *r = clib_random_buffer_get_data (&vm->random_buffer,
1598                                             sizeof (hash_seeds));
1599       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1600         hash_seeds[i] = r[i];
1601
1602       /* Mark all hash keys as been no-seen before. */
1603       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1604         hash_bitmap[i] = 0;
1605
1606       time_last_seed_change = time_now;
1607     }
1608
1609   from = vlib_frame_vector_args (frame);
1610   n_left_from = frame->n_vectors;
1611   next_index = node->cached_next_index;
1612   if (next_index == IP4_ARP_NEXT_DROP)
1613     next_index = IP4_ARP_N_NEXT;        /* point to first interface */
1614
1615   while (n_left_from > 0)
1616     {
1617       vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
1618                            to_next_drop, n_left_to_next_drop);
1619
1620       while (n_left_from > 0 && n_left_to_next_drop > 0)
1621         {
1622           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
1623           ip_adjacency_t *adj0;
1624           vlib_buffer_t *p0;
1625           ip4_header_t *ip0;
1626           uword bm0;
1627
1628           pi0 = from[0];
1629
1630           p0 = vlib_get_buffer (vm, pi0);
1631
1632           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1633           adj0 = adj_get (adj_index0);
1634           ip0 = vlib_buffer_get_current (p0);
1635
1636           a0 = hash_seeds[0];
1637           b0 = hash_seeds[1];
1638           c0 = hash_seeds[2];
1639
1640           sw_if_index0 = adj0->rewrite_header.sw_if_index;
1641           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
1642
1643           if (is_glean)
1644             {
1645               /*
1646                * this is the Glean case, so we are ARPing for the
1647                * packet's destination
1648                */
1649               a0 ^= ip0->dst_address.data_u32;
1650             }
1651           else
1652             {
1653               a0 ^= adj0->sub_type.nbr.next_hop.ip4.data_u32;
1654             }
1655           b0 ^= sw_if_index0;
1656
1657           hash_v3_mix32 (a0, b0, c0);
1658           hash_v3_finalize32 (a0, b0, c0);
1659
1660           c0 &= BITS (hash_bitmap) - 1;
1661           m0 = (uword) 1 << (c0 % BITS (uword));
1662           c0 = c0 / BITS (uword);
1663
1664           bm0 = hash_bitmap[c0];
1665           drop0 = (bm0 & m0) != 0;
1666
1667           /* Mark it as seen. */
1668           hash_bitmap[c0] = bm0 | m0;
1669
1670           from += 1;
1671           n_left_from -= 1;
1672           to_next_drop[0] = pi0;
1673           to_next_drop += 1;
1674           n_left_to_next_drop -= 1;
1675
1676           p0->error =
1677             node->errors[drop0 ? IP4_ARP_ERROR_DROP :
1678                          IP4_ARP_ERROR_REQUEST_SENT];
1679
1680           /*
1681            * the adj has been updated to a rewrite but the node the DPO that got
1682            * us here hasn't - yet. no big deal. we'll drop while we wait.
1683            */
1684           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
1685             continue;
1686
1687           if (drop0)
1688             continue;
1689
1690           /*
1691            * Can happen if the control-plane is programming tables
1692            * with traffic flowing; at least that's today's lame excuse.
1693            */
1694           if ((is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_GLEAN)
1695               || (!is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP))
1696             {
1697               p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
1698             }
1699           else
1700             /* Send ARP request. */
1701             {
1702               u32 bi0 = 0;
1703               vlib_buffer_t *b0;
1704               ethernet_arp_header_t *h0;
1705               vnet_hw_interface_t *hw_if0;
1706
1707               h0 =
1708                 vlib_packet_template_get_packet (vm,
1709                                                  &im->ip4_arp_request_packet_template,
1710                                                  &bi0);
1711
1712               /* Seems we're out of buffers */
1713               if (PREDICT_FALSE (!h0))
1714                 continue;
1715
1716               /* Add rewrite/encap string for ARP packet. */
1717               vnet_rewrite_one_header (adj0[0], h0,
1718                                        sizeof (ethernet_header_t));
1719
1720               hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1721
1722               /* Src ethernet address in ARP header. */
1723               clib_memcpy (h0->ip4_over_ethernet[0].ethernet,
1724                            hw_if0->hw_address,
1725                            sizeof (h0->ip4_over_ethernet[0].ethernet));
1726
1727               if (is_glean)
1728                 {
1729                   /* The interface's source address is stashed in the Glean Adj */
1730                   h0->ip4_over_ethernet[0].ip4 =
1731                     adj0->sub_type.glean.receive_addr.ip4;
1732
1733                   /* Copy in destination address we are requesting. This is the
1734                    * glean case, so it's the packet's destination.*/
1735                   h0->ip4_over_ethernet[1].ip4.data_u32 =
1736                     ip0->dst_address.data_u32;
1737                 }
1738               else
1739                 {
1740                   /* Src IP address in ARP header. */
1741                   if (ip4_src_address_for_packet (lm, sw_if_index0,
1742                                                   &h0->
1743                                                   ip4_over_ethernet[0].ip4))
1744                     {
1745                       /* No source address available */
1746                       p0->error =
1747                         node->errors[IP4_ARP_ERROR_NO_SOURCE_ADDRESS];
1748                       vlib_buffer_free (vm, &bi0, 1);
1749                       continue;
1750                     }
1751
1752                   /* Copy in destination address we are requesting from the
1753                      incomplete adj */
1754                   h0->ip4_over_ethernet[1].ip4.data_u32 =
1755                     adj0->sub_type.nbr.next_hop.ip4.as_u32;
1756                 }
1757
1758               vlib_buffer_copy_trace_flag (vm, p0, bi0);
1759               b0 = vlib_get_buffer (vm, bi0);
1760               VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
1761               vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
1762
1763               vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
1764
1765               vlib_set_next_frame_buffer (vm, node,
1766                                           adj0->rewrite_header.next_index,
1767                                           bi0);
1768             }
1769         }
1770
1771       vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
1772     }
1773
1774   return frame->n_vectors;
1775 }
1776
1777 static uword
1778 ip4_arp (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1779 {
1780   return (ip4_arp_inline (vm, node, frame, 0));
1781 }
1782
1783 static uword
1784 ip4_glean (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1785 {
1786   return (ip4_arp_inline (vm, node, frame, 1));
1787 }
1788
1789 static char *ip4_arp_error_strings[] = {
1790   [IP4_ARP_ERROR_DROP] = "address overflow drops",
1791   [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
1792   [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
1793   [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
1794   [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
1795   [IP4_ARP_ERROR_NO_SOURCE_ADDRESS] = "no source address for ARP request",
1796 };
1797
1798 /* *INDENT-OFF* */
1799 VLIB_REGISTER_NODE (ip4_arp_node) =
1800 {
1801   .function = ip4_arp,
1802   .name = "ip4-arp",
1803   .vector_size = sizeof (u32),
1804   .format_trace = format_ip4_forward_next_trace,
1805   .n_errors = ARRAY_LEN (ip4_arp_error_strings),
1806   .error_strings = ip4_arp_error_strings,
1807   .n_next_nodes = IP4_ARP_N_NEXT,
1808   .next_nodes =
1809   {
1810     [IP4_ARP_NEXT_DROP] = "error-drop",
1811   },
1812 };
1813
1814 VLIB_REGISTER_NODE (ip4_glean_node) =
1815 {
1816   .function = ip4_glean,
1817   .name = "ip4-glean",
1818   .vector_size = sizeof (u32),
1819   .format_trace = format_ip4_forward_next_trace,
1820   .n_errors = ARRAY_LEN (ip4_arp_error_strings),
1821   .error_strings = ip4_arp_error_strings,
1822   .n_next_nodes = IP4_ARP_N_NEXT,
1823   .next_nodes = {
1824   [IP4_ARP_NEXT_DROP] = "error-drop",
1825   },
1826 };
1827 /* *INDENT-ON* */
1828
1829 #define foreach_notrace_ip4_arp_error           \
1830 _(DROP)                                         \
1831 _(REQUEST_SENT)                                 \
1832 _(REPLICATE_DROP)                               \
1833 _(REPLICATE_FAIL)
1834
1835 clib_error_t *
1836 arp_notrace_init (vlib_main_t * vm)
1837 {
1838   vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, ip4_arp_node.index);
1839
1840   /* don't trace ARP request packets */
1841 #define _(a)                                    \
1842     vnet_pcap_drop_trace_filter_add_del         \
1843         (rt->errors[IP4_ARP_ERROR_##a],         \
1844          1 /* is_add */);
1845   foreach_notrace_ip4_arp_error;
1846 #undef _
1847   return 0;
1848 }
1849
1850 VLIB_INIT_FUNCTION (arp_notrace_init);
1851
1852
1853 /* Send an ARP request to see if given destination is reachable on given interface. */
1854 clib_error_t *
1855 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
1856 {
1857   vnet_main_t *vnm = vnet_get_main ();
1858   ip4_main_t *im = &ip4_main;
1859   ethernet_arp_header_t *h;
1860   ip4_address_t *src;
1861   ip_interface_address_t *ia;
1862   ip_adjacency_t *adj;
1863   vnet_hw_interface_t *hi;
1864   vnet_sw_interface_t *si;
1865   vlib_buffer_t *b;
1866   adj_index_t ai;
1867   u32 bi = 0;
1868
1869   si = vnet_get_sw_interface (vnm, sw_if_index);
1870
1871   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
1872     {
1873       return clib_error_return (0, "%U: interface %U down",
1874                                 format_ip4_address, dst,
1875                                 format_vnet_sw_if_index_name, vnm,
1876                                 sw_if_index);
1877     }
1878
1879   src =
1880     ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
1881   if (!src)
1882     {
1883       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
1884       return clib_error_return
1885         (0,
1886          "no matching interface address for destination %U (interface %U)",
1887          format_ip4_address, dst, format_vnet_sw_if_index_name, vnm,
1888          sw_if_index);
1889     }
1890
1891   h = vlib_packet_template_get_packet (vm,
1892                                        &im->ip4_arp_request_packet_template,
1893                                        &bi);
1894
1895   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1896   if (PREDICT_FALSE (!hi->hw_address))
1897     {
1898       return clib_error_return (0, "%U: interface %U do not support ip probe",
1899                                 format_ip4_address, dst,
1900                                 format_vnet_sw_if_index_name, vnm,
1901                                 sw_if_index);
1902     }
1903
1904   clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address,
1905                sizeof (h->ip4_over_ethernet[0].ethernet));
1906
1907   h->ip4_over_ethernet[0].ip4 = src[0];
1908   h->ip4_over_ethernet[1].ip4 = dst[0];
1909
1910   b = vlib_get_buffer (vm, bi);
1911   vnet_buffer (b)->sw_if_index[VLIB_RX] =
1912     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
1913
1914   ip46_address_t nh = {
1915     .ip4 = *dst,
1916   };
1917
1918   ai = adj_nbr_add_or_lock (FIB_PROTOCOL_IP4,
1919                             VNET_LINK_IP4, &nh, sw_if_index);
1920   adj = adj_get (ai);
1921
1922   /* Peer has been previously resolved, retrieve glean adj instead */
1923   if (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE)
1924     {
1925       adj_unlock (ai);
1926       ai = adj_glean_add_or_lock (FIB_PROTOCOL_IP4, sw_if_index, &nh);
1927       adj = adj_get (ai);
1928     }
1929
1930   /* Add encapsulation string for software interface (e.g. ethernet header). */
1931   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
1932   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
1933
1934   {
1935     vlib_frame_t *f = vlib_get_frame_to_node (vm, hi->output_node_index);
1936     u32 *to_next = vlib_frame_vector_args (f);
1937     to_next[0] = bi;
1938     f->n_vectors = 1;
1939     vlib_put_frame_to_node (vm, hi->output_node_index, f);
1940   }
1941
1942   adj_unlock (ai);
1943   return /* no error */ 0;
1944 }
1945
1946 typedef enum
1947 {
1948   IP4_REWRITE_NEXT_DROP,
1949   IP4_REWRITE_NEXT_ICMP_ERROR,
1950 } ip4_rewrite_next_t;
1951
1952 always_inline uword
1953 ip4_rewrite_inline (vlib_main_t * vm,
1954                     vlib_node_runtime_t * node,
1955                     vlib_frame_t * frame,
1956                     int do_counters, int is_midchain, int is_mcast)
1957 {
1958   ip_lookup_main_t *lm = &ip4_main.lookup_main;
1959   u32 *from = vlib_frame_vector_args (frame);
1960   u32 n_left_from, n_left_to_next, *to_next, next_index;
1961   vlib_node_runtime_t *error_node =
1962     vlib_node_get_runtime (vm, ip4_input_node.index);
1963
1964   n_left_from = frame->n_vectors;
1965   next_index = node->cached_next_index;
1966   u32 thread_index = vlib_get_thread_index ();
1967
1968   while (n_left_from > 0)
1969     {
1970       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1971
1972       while (n_left_from >= 4 && n_left_to_next >= 2)
1973         {
1974           ip_adjacency_t *adj0, *adj1;
1975           vlib_buffer_t *p0, *p1;
1976           ip4_header_t *ip0, *ip1;
1977           u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
1978           u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
1979           u32 tx_sw_if_index0, tx_sw_if_index1;
1980
1981           /* Prefetch next iteration. */
1982           {
1983             vlib_buffer_t *p2, *p3;
1984
1985             p2 = vlib_get_buffer (vm, from[2]);
1986             p3 = vlib_get_buffer (vm, from[3]);
1987
1988             vlib_prefetch_buffer_header (p2, STORE);
1989             vlib_prefetch_buffer_header (p3, STORE);
1990
1991             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1992             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1993           }
1994
1995           pi0 = to_next[0] = from[0];
1996           pi1 = to_next[1] = from[1];
1997
1998           from += 2;
1999           n_left_from -= 2;
2000           to_next += 2;
2001           n_left_to_next -= 2;
2002
2003           p0 = vlib_get_buffer (vm, pi0);
2004           p1 = vlib_get_buffer (vm, pi1);
2005
2006           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2007           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
2008
2009           /*
2010            * pre-fetch the per-adjacency counters
2011            */
2012           if (do_counters)
2013             {
2014               vlib_prefetch_combined_counter (&adjacency_counters,
2015                                               thread_index, adj_index0);
2016               vlib_prefetch_combined_counter (&adjacency_counters,
2017                                               thread_index, adj_index1);
2018             }
2019
2020           ip0 = vlib_buffer_get_current (p0);
2021           ip1 = vlib_buffer_get_current (p1);
2022
2023           error0 = error1 = IP4_ERROR_NONE;
2024           next0 = next1 = IP4_REWRITE_NEXT_DROP;
2025
2026           /* Decrement TTL & update checksum.
2027              Works either endian, so no need for byte swap. */
2028           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2029             {
2030               i32 ttl0 = ip0->ttl;
2031
2032               /* Input node should have reject packets with ttl 0. */
2033               ASSERT (ip0->ttl > 0);
2034
2035               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2036               checksum0 += checksum0 >= 0xffff;
2037
2038               ip0->checksum = checksum0;
2039               ttl0 -= 1;
2040               ip0->ttl = ttl0;
2041
2042               /*
2043                * If the ttl drops below 1 when forwarding, generate
2044                * an ICMP response.
2045                */
2046               if (PREDICT_FALSE (ttl0 <= 0))
2047                 {
2048                   error0 = IP4_ERROR_TIME_EXPIRED;
2049                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2050                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2051                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2052                                                0);
2053                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2054                 }
2055
2056               /* Verify checksum. */
2057               ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
2058                       (p0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2059             }
2060           else
2061             {
2062               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2063             }
2064           if (PREDICT_TRUE (!(p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2065             {
2066               i32 ttl1 = ip1->ttl;
2067
2068               /* Input node should have reject packets with ttl 0. */
2069               ASSERT (ip1->ttl > 0);
2070
2071               checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2072               checksum1 += checksum1 >= 0xffff;
2073
2074               ip1->checksum = checksum1;
2075               ttl1 -= 1;
2076               ip1->ttl = ttl1;
2077
2078               /*
2079                * If the ttl drops below 1 when forwarding, generate
2080                * an ICMP response.
2081                */
2082               if (PREDICT_FALSE (ttl1 <= 0))
2083                 {
2084                   error1 = IP4_ERROR_TIME_EXPIRED;
2085                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2086                   icmp4_error_set_vnet_buffer (p1, ICMP4_time_exceeded,
2087                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2088                                                0);
2089                   next1 = IP4_REWRITE_NEXT_ICMP_ERROR;
2090                 }
2091
2092               /* Verify checksum. */
2093               ASSERT ((ip1->checksum == ip4_header_checksum (ip1)) ||
2094                       (p1->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2095             }
2096           else
2097             {
2098               p1->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2099             }
2100
2101           /* Rewrite packet header and updates lengths. */
2102           adj0 = adj_get (adj_index0);
2103           adj1 = adj_get (adj_index1);
2104
2105           /* Worth pipelining. No guarantee that adj0,1 are hot... */
2106           rw_len0 = adj0[0].rewrite_header.data_bytes;
2107           rw_len1 = adj1[0].rewrite_header.data_bytes;
2108           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2109           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
2110
2111           /* Check MTU of outgoing interface. */
2112           if (vlib_buffer_length_in_chain (vm, p0) >
2113               adj0[0].rewrite_header.max_l3_packet_bytes)
2114             {
2115               error0 = IP4_ERROR_MTU_EXCEEDED;
2116               next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2117               icmp4_error_set_vnet_buffer
2118                 (p0, ICMP4_destination_unreachable,
2119                  ICMP4_destination_unreachable_fragmentation_needed_and_dont_fragment_set,
2120                  0);
2121             }
2122           if (vlib_buffer_length_in_chain (vm, p1) >
2123               adj1[0].rewrite_header.max_l3_packet_bytes)
2124             {
2125               error1 = IP4_ERROR_MTU_EXCEEDED;
2126               next1 = IP4_REWRITE_NEXT_ICMP_ERROR;
2127               icmp4_error_set_vnet_buffer
2128                 (p1, ICMP4_destination_unreachable,
2129                  ICMP4_destination_unreachable_fragmentation_needed_and_dont_fragment_set,
2130                  0);
2131             }
2132
2133           if (is_mcast)
2134             {
2135               error0 = ((adj0[0].rewrite_header.sw_if_index ==
2136                          vnet_buffer (p0)->sw_if_index[VLIB_RX]) ?
2137                         IP4_ERROR_SAME_INTERFACE : error0);
2138               error1 = ((adj1[0].rewrite_header.sw_if_index ==
2139                          vnet_buffer (p1)->sw_if_index[VLIB_RX]) ?
2140                         IP4_ERROR_SAME_INTERFACE : error1);
2141             }
2142
2143           p0->error = error_node->errors[error0];
2144           p1->error = error_node->errors[error1];
2145           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2146            * to see the IP headerr */
2147           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2148             {
2149               next0 = adj0[0].rewrite_header.next_index;
2150               p0->current_data -= rw_len0;
2151               p0->current_length += rw_len0;
2152               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2153               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2154
2155               if (PREDICT_FALSE
2156                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2157                 vnet_feature_arc_start (lm->output_feature_arc_index,
2158                                         tx_sw_if_index0, &next0, p0);
2159             }
2160           if (PREDICT_TRUE (error1 == IP4_ERROR_NONE))
2161             {
2162               next1 = adj1[0].rewrite_header.next_index;
2163               p1->current_data -= rw_len1;
2164               p1->current_length += rw_len1;
2165
2166               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
2167               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
2168
2169               if (PREDICT_FALSE
2170                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2171                 vnet_feature_arc_start (lm->output_feature_arc_index,
2172                                         tx_sw_if_index1, &next1, p1);
2173             }
2174
2175           /* Guess we are only writing on simple Ethernet header. */
2176           vnet_rewrite_two_headers (adj0[0], adj1[0],
2177                                     ip0, ip1, sizeof (ethernet_header_t));
2178
2179           /*
2180            * Bump the per-adjacency counters
2181            */
2182           if (do_counters)
2183             {
2184               vlib_increment_combined_counter
2185                 (&adjacency_counters,
2186                  thread_index,
2187                  adj_index0, 1,
2188                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2189
2190               vlib_increment_combined_counter
2191                 (&adjacency_counters,
2192                  thread_index,
2193                  adj_index1, 1,
2194                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
2195             }
2196
2197           if (is_midchain)
2198             {
2199               adj0->sub_type.midchain.fixup_func
2200                 (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
2201               adj1->sub_type.midchain.fixup_func
2202                 (vm, adj1, p1, adj0->sub_type.midchain.fixup_data);
2203             }
2204           if (is_mcast)
2205             {
2206               /*
2207                * copy bytes from the IP address into the MAC rewrite
2208                */
2209               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2210               vnet_fixup_one_header (adj1[0], &ip1->dst_address, ip1);
2211             }
2212
2213           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2214                                            to_next, n_left_to_next,
2215                                            pi0, pi1, next0, next1);
2216         }
2217
2218       while (n_left_from > 0 && n_left_to_next > 0)
2219         {
2220           ip_adjacency_t *adj0;
2221           vlib_buffer_t *p0;
2222           ip4_header_t *ip0;
2223           u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2224           u32 tx_sw_if_index0;
2225
2226           pi0 = to_next[0] = from[0];
2227
2228           p0 = vlib_get_buffer (vm, pi0);
2229
2230           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2231
2232           adj0 = adj_get (adj_index0);
2233
2234           ip0 = vlib_buffer_get_current (p0);
2235
2236           error0 = IP4_ERROR_NONE;
2237           next0 = IP4_REWRITE_NEXT_DROP;        /* drop on error */
2238
2239           /* Decrement TTL & update checksum. */
2240           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2241             {
2242               i32 ttl0 = ip0->ttl;
2243
2244               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2245
2246               checksum0 += checksum0 >= 0xffff;
2247
2248               ip0->checksum = checksum0;
2249
2250               ASSERT (ip0->ttl > 0);
2251
2252               ttl0 -= 1;
2253
2254               ip0->ttl = ttl0;
2255
2256               ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
2257                       (p0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2258
2259               if (PREDICT_FALSE (ttl0 <= 0))
2260                 {
2261                   /*
2262                    * If the ttl drops below 1 when forwarding, generate
2263                    * an ICMP response.
2264                    */
2265                   error0 = IP4_ERROR_TIME_EXPIRED;
2266                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2267                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2268                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2269                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2270                                                0);
2271                 }
2272             }
2273           else
2274             {
2275               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2276             }
2277
2278           if (do_counters)
2279             vlib_prefetch_combined_counter (&adjacency_counters,
2280                                             thread_index, adj_index0);
2281
2282           /* Guess we are only writing on simple Ethernet header. */
2283           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2284           if (is_mcast)
2285             {
2286               /*
2287                * copy bytes from the IP address into the MAC rewrite
2288                */
2289               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2290             }
2291
2292           /* Update packet buffer attributes/set output interface. */
2293           rw_len0 = adj0[0].rewrite_header.data_bytes;
2294           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2295
2296           if (do_counters)
2297             vlib_increment_combined_counter
2298               (&adjacency_counters,
2299                thread_index, adj_index0, 1,
2300                vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2301
2302           /* Check MTU of outgoing interface. */
2303           if (vlib_buffer_length_in_chain (vm, p0) >
2304               adj0[0].rewrite_header.max_l3_packet_bytes)
2305             {
2306               error0 = IP4_ERROR_MTU_EXCEEDED;
2307               next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2308               icmp4_error_set_vnet_buffer
2309                 (p0, ICMP4_destination_unreachable,
2310                  ICMP4_destination_unreachable_fragmentation_needed_and_dont_fragment_set,
2311                  0);
2312             }
2313           if (is_mcast)
2314             {
2315               error0 = ((adj0[0].rewrite_header.sw_if_index ==
2316                          vnet_buffer (p0)->sw_if_index[VLIB_RX]) ?
2317                         IP4_ERROR_SAME_INTERFACE : error0);
2318             }
2319           p0->error = error_node->errors[error0];
2320
2321           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2322            * to see the IP headerr */
2323           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2324             {
2325               p0->current_data -= rw_len0;
2326               p0->current_length += rw_len0;
2327               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2328
2329               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2330               next0 = adj0[0].rewrite_header.next_index;
2331
2332               if (is_midchain)
2333                 {
2334                   adj0->sub_type.midchain.fixup_func
2335                     (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
2336                 }
2337
2338               if (PREDICT_FALSE
2339                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2340                 vnet_feature_arc_start (lm->output_feature_arc_index,
2341                                         tx_sw_if_index0, &next0, p0);
2342
2343             }
2344
2345           from += 1;
2346           n_left_from -= 1;
2347           to_next += 1;
2348           n_left_to_next -= 1;
2349
2350           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2351                                            to_next, n_left_to_next,
2352                                            pi0, next0);
2353         }
2354
2355       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2356     }
2357
2358   /* Need to do trace after rewrites to pick up new packet data. */
2359   if (node->flags & VLIB_NODE_FLAG_TRACE)
2360     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2361
2362   return frame->n_vectors;
2363 }
2364
2365
2366 /** @brief IPv4 rewrite node.
2367     @node ip4-rewrite
2368
2369     This is the IPv4 transit-rewrite node: decrement TTL, fix the ipv4
2370     header checksum, fetch the ip adjacency, check the outbound mtu,
2371     apply the adjacency rewrite, and send pkts to the adjacency
2372     rewrite header's rewrite_next_index.
2373
2374     @param vm vlib_main_t corresponding to the current thread
2375     @param node vlib_node_runtime_t
2376     @param frame vlib_frame_t whose contents should be dispatched
2377
2378     @par Graph mechanics: buffer metadata, next index usage
2379
2380     @em Uses:
2381     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
2382         - the rewrite adjacency index
2383     - <code>adj->lookup_next_index</code>
2384         - Must be IP_LOOKUP_NEXT_REWRITE or IP_LOOKUP_NEXT_ARP, otherwise
2385           the packet will be dropped.
2386     - <code>adj->rewrite_header</code>
2387         - Rewrite string length, rewrite string, next_index
2388
2389     @em Sets:
2390     - <code>b->current_data, b->current_length</code>
2391         - Updated net of applying the rewrite string
2392
2393     <em>Next Indices:</em>
2394     - <code> adj->rewrite_header.next_index </code>
2395       or @c ip4-drop
2396 */
2397 static uword
2398 ip4_rewrite (vlib_main_t * vm,
2399              vlib_node_runtime_t * node, vlib_frame_t * frame)
2400 {
2401   if (adj_are_counters_enabled ())
2402     return ip4_rewrite_inline (vm, node, frame, 1, 0, 0);
2403   else
2404     return ip4_rewrite_inline (vm, node, frame, 0, 0, 0);
2405 }
2406
2407 static uword
2408 ip4_midchain (vlib_main_t * vm,
2409               vlib_node_runtime_t * node, vlib_frame_t * frame)
2410 {
2411   if (adj_are_counters_enabled ())
2412     return ip4_rewrite_inline (vm, node, frame, 1, 1, 0);
2413   else
2414     return ip4_rewrite_inline (vm, node, frame, 0, 1, 0);
2415 }
2416
2417 static uword
2418 ip4_rewrite_mcast (vlib_main_t * vm,
2419                    vlib_node_runtime_t * node, vlib_frame_t * frame)
2420 {
2421   if (adj_are_counters_enabled ())
2422     return ip4_rewrite_inline (vm, node, frame, 1, 0, 1);
2423   else
2424     return ip4_rewrite_inline (vm, node, frame, 0, 0, 1);
2425 }
2426
2427 static uword
2428 ip4_mcast_midchain (vlib_main_t * vm,
2429                     vlib_node_runtime_t * node, vlib_frame_t * frame)
2430 {
2431   if (adj_are_counters_enabled ())
2432     return ip4_rewrite_inline (vm, node, frame, 1, 1, 1);
2433   else
2434     return ip4_rewrite_inline (vm, node, frame, 0, 1, 1);
2435 }
2436
2437 /* *INDENT-OFF* */
2438 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2439   .function = ip4_rewrite,
2440   .name = "ip4-rewrite",
2441   .vector_size = sizeof (u32),
2442
2443   .format_trace = format_ip4_rewrite_trace,
2444
2445   .n_next_nodes = 2,
2446   .next_nodes = {
2447     [IP4_REWRITE_NEXT_DROP] = "ip4-drop",
2448     [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2449   },
2450 };
2451 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite)
2452
2453 VLIB_REGISTER_NODE (ip4_rewrite_mcast_node) = {
2454   .function = ip4_rewrite_mcast,
2455   .name = "ip4-rewrite-mcast",
2456   .vector_size = sizeof (u32),
2457
2458   .format_trace = format_ip4_rewrite_trace,
2459   .sibling_of = "ip4-rewrite",
2460 };
2461 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_mcast_node, ip4_rewrite_mcast)
2462
2463 VLIB_REGISTER_NODE (ip4_mcast_midchain_node, static) = {
2464   .function = ip4_mcast_midchain,
2465   .name = "ip4-mcast-midchain",
2466   .vector_size = sizeof (u32),
2467
2468   .format_trace = format_ip4_rewrite_trace,
2469   .sibling_of = "ip4-rewrite",
2470 };
2471 VLIB_NODE_FUNCTION_MULTIARCH (ip4_mcast_midchain_node, ip4_mcast_midchain)
2472
2473 VLIB_REGISTER_NODE (ip4_midchain_node) = {
2474   .function = ip4_midchain,
2475   .name = "ip4-midchain",
2476   .vector_size = sizeof (u32),
2477   .format_trace = format_ip4_forward_next_trace,
2478   .sibling_of =  "ip4-rewrite",
2479 };
2480 VLIB_NODE_FUNCTION_MULTIARCH (ip4_midchain_node, ip4_midchain);
2481 /* *INDENT-ON */
2482
2483 int
2484 ip4_lookup_validate (ip4_address_t * a, u32 fib_index0)
2485 {
2486   ip4_fib_mtrie_t *mtrie0;
2487   ip4_fib_mtrie_leaf_t leaf0;
2488   u32 lbi0;
2489
2490   mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
2491
2492   leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, a);
2493   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
2494   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
2495
2496   lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2497
2498   return lbi0 == ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0), a);
2499 }
2500
2501 static clib_error_t *
2502 test_lookup_command_fn (vlib_main_t * vm,
2503                         unformat_input_t * input, vlib_cli_command_t * cmd)
2504 {
2505   ip4_fib_t *fib;
2506   u32 table_id = 0;
2507   f64 count = 1;
2508   u32 n;
2509   int i;
2510   ip4_address_t ip4_base_address;
2511   u64 errors = 0;
2512
2513   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2514     {
2515       if (unformat (input, "table %d", &table_id))
2516         {
2517           /* Make sure the entry exists. */
2518           fib = ip4_fib_get (table_id);
2519           if ((fib) && (fib->index != table_id))
2520             return clib_error_return (0, "<fib-index> %d does not exist",
2521                                       table_id);
2522         }
2523       else if (unformat (input, "count %f", &count))
2524         ;
2525
2526       else if (unformat (input, "%U",
2527                          unformat_ip4_address, &ip4_base_address))
2528         ;
2529       else
2530         return clib_error_return (0, "unknown input `%U'",
2531                                   format_unformat_error, input);
2532     }
2533
2534   n = count;
2535
2536   for (i = 0; i < n; i++)
2537     {
2538       if (!ip4_lookup_validate (&ip4_base_address, table_id))
2539         errors++;
2540
2541       ip4_base_address.as_u32 =
2542         clib_host_to_net_u32 (1 +
2543                               clib_net_to_host_u32 (ip4_base_address.as_u32));
2544     }
2545
2546   if (errors)
2547     vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
2548   else
2549     vlib_cli_output (vm, "No errors in %d lookups\n", n);
2550
2551   return 0;
2552 }
2553
2554 /*?
2555  * Perform a lookup of an IPv4 Address (or range of addresses) in the
2556  * given FIB table to determine if there is a conflict with the
2557  * adjacency table. The fib-id can be determined by using the
2558  * '<em>show ip fib</em>' command. If fib-id is not entered, default value
2559  * of 0 is used.
2560  *
2561  * @todo This command uses fib-id, other commands use table-id (not
2562  * just a name, they are different indexes). Would like to change this
2563  * to table-id for consistency.
2564  *
2565  * @cliexpar
2566  * Example of how to run the test lookup command:
2567  * @cliexstart{test lookup 172.16.1.1 table 1 count 2}
2568  * No errors in 2 lookups
2569  * @cliexend
2570 ?*/
2571 /* *INDENT-OFF* */
2572 VLIB_CLI_COMMAND (lookup_test_command, static) =
2573 {
2574   .path = "test lookup",
2575   .short_help = "test lookup <ipv4-addr> [table <fib-id>] [count <nn>]",
2576   .function = test_lookup_command_fn,
2577 };
2578 /* *INDENT-ON* */
2579
2580 int
2581 vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
2582 {
2583   u32 fib_index;
2584
2585   fib_index = fib_table_find (FIB_PROTOCOL_IP4, table_id);
2586
2587   if (~0 == fib_index)
2588     return VNET_API_ERROR_NO_SUCH_FIB;
2589
2590   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP4,
2591                                   flow_hash_config);
2592
2593   return 0;
2594 }
2595
2596 static clib_error_t *
2597 set_ip_flow_hash_command_fn (vlib_main_t * vm,
2598                              unformat_input_t * input,
2599                              vlib_cli_command_t * cmd)
2600 {
2601   int matched = 0;
2602   u32 table_id = 0;
2603   u32 flow_hash_config = 0;
2604   int rv;
2605
2606   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2607     {
2608       if (unformat (input, "table %d", &table_id))
2609         matched = 1;
2610 #define _(a,v) \
2611     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2612       foreach_flow_hash_bit
2613 #undef _
2614         else
2615         break;
2616     }
2617
2618   if (matched == 0)
2619     return clib_error_return (0, "unknown input `%U'",
2620                               format_unformat_error, input);
2621
2622   rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
2623   switch (rv)
2624     {
2625     case 0:
2626       break;
2627
2628     case VNET_API_ERROR_NO_SUCH_FIB:
2629       return clib_error_return (0, "no such FIB table %d", table_id);
2630
2631     default:
2632       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2633       break;
2634     }
2635
2636   return 0;
2637 }
2638
2639 /*?
2640  * Configure the set of IPv4 fields used by the flow hash.
2641  *
2642  * @cliexpar
2643  * Example of how to set the flow hash on a given table:
2644  * @cliexcmd{set ip flow-hash table 7 dst sport dport proto}
2645  * Example of display the configured flow hash:
2646  * @cliexstart{show ip fib}
2647  * ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2648  * 0.0.0.0/0
2649  *   unicast-ip4-chain
2650  *   [@0]: dpo-load-balance: [index:0 buckets:1 uRPF:0 to:[0:0]]
2651  *     [0] [@0]: dpo-drop ip6
2652  * 0.0.0.0/32
2653  *   unicast-ip4-chain
2654  *   [@0]: dpo-load-balance: [index:1 buckets:1 uRPF:1 to:[0:0]]
2655  *     [0] [@0]: dpo-drop ip6
2656  * 224.0.0.0/8
2657  *   unicast-ip4-chain
2658  *   [@0]: dpo-load-balance: [index:3 buckets:1 uRPF:3 to:[0:0]]
2659  *     [0] [@0]: dpo-drop ip6
2660  * 6.0.1.2/32
2661  *   unicast-ip4-chain
2662  *   [@0]: dpo-load-balance: [index:30 buckets:1 uRPF:29 to:[0:0]]
2663  *     [0] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
2664  * 7.0.0.1/32
2665  *   unicast-ip4-chain
2666  *   [@0]: dpo-load-balance: [index:31 buckets:4 uRPF:30 to:[0:0]]
2667  *     [0] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
2668  *     [1] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
2669  *     [2] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
2670  *     [3] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
2671  * 240.0.0.0/8
2672  *   unicast-ip4-chain
2673  *   [@0]: dpo-load-balance: [index:2 buckets:1 uRPF:2 to:[0:0]]
2674  *     [0] [@0]: dpo-drop ip6
2675  * 255.255.255.255/32
2676  *   unicast-ip4-chain
2677  *   [@0]: dpo-load-balance: [index:4 buckets:1 uRPF:4 to:[0:0]]
2678  *     [0] [@0]: dpo-drop ip6
2679  * ipv4-VRF:7, fib_index 1, flow hash: dst sport dport proto
2680  * 0.0.0.0/0
2681  *   unicast-ip4-chain
2682  *   [@0]: dpo-load-balance: [index:12 buckets:1 uRPF:11 to:[0:0]]
2683  *     [0] [@0]: dpo-drop ip6
2684  * 0.0.0.0/32
2685  *   unicast-ip4-chain
2686  *   [@0]: dpo-load-balance: [index:13 buckets:1 uRPF:12 to:[0:0]]
2687  *     [0] [@0]: dpo-drop ip6
2688  * 172.16.1.0/24
2689  *   unicast-ip4-chain
2690  *   [@0]: dpo-load-balance: [index:17 buckets:1 uRPF:16 to:[0:0]]
2691  *     [0] [@4]: ipv4-glean: af_packet0
2692  * 172.16.1.1/32
2693  *   unicast-ip4-chain
2694  *   [@0]: dpo-load-balance: [index:18 buckets:1 uRPF:17 to:[1:84]]
2695  *     [0] [@2]: dpo-receive: 172.16.1.1 on af_packet0
2696  * 172.16.1.2/32
2697  *   unicast-ip4-chain
2698  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
2699  *     [0] [@5]: ipv4 via 172.16.1.2 af_packet0: IP4: 02:fe:9e:70:7a:2b -> 26:a5:f6:9c:3a:36
2700  * 172.16.2.0/24
2701  *   unicast-ip4-chain
2702  *   [@0]: dpo-load-balance: [index:19 buckets:1 uRPF:18 to:[0:0]]
2703  *     [0] [@4]: ipv4-glean: af_packet1
2704  * 172.16.2.1/32
2705  *   unicast-ip4-chain
2706  *   [@0]: dpo-load-balance: [index:20 buckets:1 uRPF:19 to:[0:0]]
2707  *     [0] [@2]: dpo-receive: 172.16.2.1 on af_packet1
2708  * 224.0.0.0/8
2709  *   unicast-ip4-chain
2710  *   [@0]: dpo-load-balance: [index:15 buckets:1 uRPF:14 to:[0:0]]
2711  *     [0] [@0]: dpo-drop ip6
2712  * 240.0.0.0/8
2713  *   unicast-ip4-chain
2714  *   [@0]: dpo-load-balance: [index:14 buckets:1 uRPF:13 to:[0:0]]
2715  *     [0] [@0]: dpo-drop ip6
2716  * 255.255.255.255/32
2717  *   unicast-ip4-chain
2718  *   [@0]: dpo-load-balance: [index:16 buckets:1 uRPF:15 to:[0:0]]
2719  *     [0] [@0]: dpo-drop ip6
2720  * @cliexend
2721 ?*/
2722 /* *INDENT-OFF* */
2723 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) =
2724 {
2725   .path = "set ip flow-hash",
2726   .short_help =
2727   "set ip flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
2728   .function = set_ip_flow_hash_command_fn,
2729 };
2730 /* *INDENT-ON* */
2731
2732 int
2733 vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
2734                              u32 table_index)
2735 {
2736   vnet_main_t *vnm = vnet_get_main ();
2737   vnet_interface_main_t *im = &vnm->interface_main;
2738   ip4_main_t *ipm = &ip4_main;
2739   ip_lookup_main_t *lm = &ipm->lookup_main;
2740   vnet_classify_main_t *cm = &vnet_classify_main;
2741   ip4_address_t *if_addr;
2742
2743   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
2744     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
2745
2746   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
2747     return VNET_API_ERROR_NO_SUCH_ENTRY;
2748
2749   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
2750   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
2751
2752   if_addr = ip4_interface_first_address (ipm, sw_if_index, NULL);
2753
2754   if (NULL != if_addr)
2755     {
2756       fib_prefix_t pfx = {
2757         .fp_len = 32,
2758         .fp_proto = FIB_PROTOCOL_IP4,
2759         .fp_addr.ip4 = *if_addr,
2760       };
2761       u32 fib_index;
2762
2763       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2764                                                        sw_if_index);
2765
2766
2767       if (table_index != (u32) ~ 0)
2768         {
2769           dpo_id_t dpo = DPO_INVALID;
2770
2771           dpo_set (&dpo,
2772                    DPO_CLASSIFY,
2773                    DPO_PROTO_IP4,
2774                    classify_dpo_create (DPO_PROTO_IP4, table_index));
2775
2776           fib_table_entry_special_dpo_add (fib_index,
2777                                            &pfx,
2778                                            FIB_SOURCE_CLASSIFY,
2779                                            FIB_ENTRY_FLAG_NONE, &dpo);
2780           dpo_reset (&dpo);
2781         }
2782       else
2783         {
2784           fib_table_entry_special_remove (fib_index,
2785                                           &pfx, FIB_SOURCE_CLASSIFY);
2786         }
2787     }
2788
2789   return 0;
2790 }
2791
2792 static clib_error_t *
2793 set_ip_classify_command_fn (vlib_main_t * vm,
2794                             unformat_input_t * input,
2795                             vlib_cli_command_t * cmd)
2796 {
2797   u32 table_index = ~0;
2798   int table_index_set = 0;
2799   u32 sw_if_index = ~0;
2800   int rv;
2801
2802   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2803     {
2804       if (unformat (input, "table-index %d", &table_index))
2805         table_index_set = 1;
2806       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
2807                          vnet_get_main (), &sw_if_index))
2808         ;
2809       else
2810         break;
2811     }
2812
2813   if (table_index_set == 0)
2814     return clib_error_return (0, "classify table-index must be specified");
2815
2816   if (sw_if_index == ~0)
2817     return clib_error_return (0, "interface / subif must be specified");
2818
2819   rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
2820
2821   switch (rv)
2822     {
2823     case 0:
2824       break;
2825
2826     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
2827       return clib_error_return (0, "No such interface");
2828
2829     case VNET_API_ERROR_NO_SUCH_ENTRY:
2830       return clib_error_return (0, "No such classifier table");
2831     }
2832   return 0;
2833 }
2834
2835 /*?
2836  * Assign a classification table to an interface. The classification
2837  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
2838  * commands. Once the table is create, use this command to filter packets
2839  * on an interface.
2840  *
2841  * @cliexpar
2842  * Example of how to assign a classification table to an interface:
2843  * @cliexcmd{set ip classify intfc GigabitEthernet2/0/0 table-index 1}
2844 ?*/
2845 /* *INDENT-OFF* */
2846 VLIB_CLI_COMMAND (set_ip_classify_command, static) =
2847 {
2848     .path = "set ip classify",
2849     .short_help =
2850     "set ip classify intfc <interface> table-index <classify-idx>",
2851     .function = set_ip_classify_command_fn,
2852 };
2853 /* *INDENT-ON* */
2854
2855 static clib_error_t *
2856 ip4_config (vlib_main_t * vm, unformat_input_t * input)
2857 {
2858   ip4_main_t *im = &ip4_main;
2859   uword heapsize = 0;
2860
2861   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2862     {
2863       if (unformat (input, "heap-size %U", unformat_memory_size, &heapsize))
2864         ;
2865       else
2866         return clib_error_return (0,
2867                                   "invalid heap-size parameter `%U'",
2868                                   format_unformat_error, input);
2869     }
2870
2871   im->mtrie_heap_size = heapsize;
2872
2873   return 0;
2874 }
2875
2876 VLIB_EARLY_CONFIG_FUNCTION (ip4_config, "ip");
2877
2878 /*
2879  * fd.io coding-style-patch-verification: ON
2880  *
2881  * Local Variables:
2882  * eval: (c-set-style "gnu")
2883  * End:
2884  */