src/vnet/ip/ip4_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip4_forward.c: IP v4 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  43 #include <vnet/ethernet/arp_packet.h>   /* for ethernet_arp_header_t */
  44 #include <vnet/ppp/ppp.h>
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vnet/api_errno.h>     /* for API error numbers */
  47 #include <vnet/fib/fib_table.h> /* for FIB table and entry creation */
  48 #include <vnet/fib/fib_entry.h> /* for FIB table and entry creation */
  49 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  50 #include <vnet/fib/ip4_fib.h>
  51 #include <vnet/dpo/load_balance.h>
  52 #include <vnet/dpo/load_balance_map.h>
  53 #include <vnet/dpo/classify_dpo.h>
  54 #include <vnet/mfib/mfib_table.h>       /* for mFIB table and entry creation */
  55
  56 #include <vnet/ip/ip4_forward.h>
  57
  58 /** @brief IPv4 lookup node.
  59     @node ip4-lookup
  60
  61     This is the main IPv4 lookup dispatch node.
  62
  63     @param vm vlib_main_t corresponding to the current thread
  64     @param node vlib_node_runtime_t
  65     @param frame vlib_frame_t whose contents should be dispatched
  66
  67     @par Graph mechanics: buffer metadata, next index usage
  68
  69     @em Uses:
  70     - <code>vnet_buffer(b)->sw_if_index[VLIB_RX]</code>
  71         - Indicates the @c sw_if_index value of the interface that the
  72           packet was received on.
  73     - <code>vnet_buffer(b)->sw_if_index[VLIB_TX]</code>
  74         - When the value is @c ~0 then the node performs a longest prefix
  75           match (LPM) for the packet destination address in the FIB attached
  76           to the receive interface.
  77         - Otherwise perform LPM for the packet destination address in the
  78           indicated FIB. In this case <code>[VLIB_TX]</code> is a FIB index
  79           value (0, 1, ...) and not a VRF id.
  80
  81     @em Sets:
  82     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
  83         - The lookup result adjacency index.
  84
  85     <em>Next Index:</em>
  86     - Dispatches the packet to the node index found in
  87       ip_adjacency_t @c adj->lookup_next_index
  88       (where @c adj is the lookup result adjacency).
  89 */
  90 static uword
  91 ip4_lookup (vlib_main_t * vm,
  92             vlib_node_runtime_t * node, vlib_frame_t * frame)
  93 {
  94   return ip4_lookup_inline (vm, node, frame,
  95                             /* lookup_for_responses_to_locally_received_packets */
  96                             0);
  97
  98 }
  99
 100 static u8 *format_ip4_lookup_trace (u8 * s, va_list * args);
 101
 102 /* *INDENT-OFF* */
 103 VLIB_REGISTER_NODE (ip4_lookup_node) =
 104 {
 105   .function = ip4_lookup,
 106   .name = "ip4-lookup",
 107   .vector_size = sizeof (u32),
 108   .format_trace = format_ip4_lookup_trace,
 109   .n_next_nodes = IP_LOOKUP_N_NEXT,
 110   .next_nodes = IP4_LOOKUP_NEXT_NODES,
 111 };
 112 /* *INDENT-ON* */
 113
 114 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup);
 115
 116 always_inline uword
 117 ip4_load_balance (vlib_main_t * vm,
 118                   vlib_node_runtime_t * node, vlib_frame_t * frame)
 119 {
 120   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 121   u32 n_left_from, n_left_to_next, *from, *to_next;
 122   ip_lookup_next_t next;
 123   u32 thread_index = vlib_get_thread_index ();
 124
 125   from = vlib_frame_vector_args (frame);
 126   n_left_from = frame->n_vectors;
 127   next = node->cached_next_index;
 128
 129   if (node->flags & VLIB_NODE_FLAG_TRACE)
 130     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 131
 132   while (n_left_from > 0)
 133     {
 134       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 135
 136
 137       while (n_left_from >= 4 && n_left_to_next >= 2)
 138         {
 139           ip_lookup_next_t next0, next1;
 140           const load_balance_t *lb0, *lb1;
 141           vlib_buffer_t *p0, *p1;
 142           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 143           const ip4_header_t *ip0, *ip1;
 144           const dpo_id_t *dpo0, *dpo1;
 145
 146           /* Prefetch next iteration. */
 147           {
 148             vlib_buffer_t *p2, *p3;
 149
 150             p2 = vlib_get_buffer (vm, from[2]);
 151             p3 = vlib_get_buffer (vm, from[3]);
 152
 153             vlib_prefetch_buffer_header (p2, STORE);
 154             vlib_prefetch_buffer_header (p3, STORE);
 155
 156             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 157             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 158           }
 159
 160           pi0 = to_next[0] = from[0];
 161           pi1 = to_next[1] = from[1];
 162
 163           from += 2;
 164           n_left_from -= 2;
 165           to_next += 2;
 166           n_left_to_next -= 2;
 167
 168           p0 = vlib_get_buffer (vm, pi0);
 169           p1 = vlib_get_buffer (vm, pi1);
 170
 171           ip0 = vlib_buffer_get_current (p0);
 172           ip1 = vlib_buffer_get_current (p1);
 173           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 174           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 175
 176           lb0 = load_balance_get (lbi0);
 177           lb1 = load_balance_get (lbi1);
 178
 179           /*
 180            * this node is for via FIBs we can re-use the hash value from the
 181            * to node if present.
 182            * We don't want to use the same hash value at each level in the recursion
 183            * graph as that would lead to polarisation
 184            */
 185           hc0 = hc1 = 0;
 186
 187           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 188             {
 189               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 190                 {
 191                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 192                     vnet_buffer (p0)->ip.flow_hash >> 1;
 193                 }
 194               else
 195                 {
 196                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 197                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 198                 }
 199               dpo0 = load_balance_get_fwd_bucket
 200                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 201             }
 202           else
 203             {
 204               dpo0 = load_balance_get_bucket_i (lb0, 0);
 205             }
 206           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 207             {
 208               if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash))
 209                 {
 210                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 211                     vnet_buffer (p1)->ip.flow_hash >> 1;
 212                 }
 213               else
 214                 {
 215                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 216                     ip4_compute_flow_hash (ip1, lb1->lb_hash_config);
 217                 }
 218               dpo1 = load_balance_get_fwd_bucket
 219                 (lb1, (hc1 & (lb1->lb_n_buckets_minus_1)));
 220             }
 221           else
 222             {
 223               dpo1 = load_balance_get_bucket_i (lb1, 0);
 224             }
 225
 226           next0 = dpo0->dpoi_next_node;
 227           next1 = dpo1->dpoi_next_node;
 228
 229           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 230           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 231
 232           vlib_increment_combined_counter
 233             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 234           vlib_increment_combined_counter
 235             (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 236
 237           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 238                                            to_next, n_left_to_next,
 239                                            pi0, pi1, next0, next1);
 240         }
 241
 242       while (n_left_from > 0 && n_left_to_next > 0)
 243         {
 244           ip_lookup_next_t next0;
 245           const load_balance_t *lb0;
 246           vlib_buffer_t *p0;
 247           u32 pi0, lbi0, hc0;
 248           const ip4_header_t *ip0;
 249           const dpo_id_t *dpo0;
 250
 251           pi0 = from[0];
 252           to_next[0] = pi0;
 253           from += 1;
 254           to_next += 1;
 255           n_left_to_next -= 1;
 256           n_left_from -= 1;
 257
 258           p0 = vlib_get_buffer (vm, pi0);
 259
 260           ip0 = vlib_buffer_get_current (p0);
 261           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 262
 263           lb0 = load_balance_get (lbi0);
 264
 265           hc0 = 0;
 266           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 267             {
 268               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 269                 {
 270                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 271                     vnet_buffer (p0)->ip.flow_hash >> 1;
 272                 }
 273               else
 274                 {
 275                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 276                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 277                 }
 278               dpo0 = load_balance_get_fwd_bucket
 279                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 280             }
 281           else
 282             {
 283               dpo0 = load_balance_get_bucket_i (lb0, 0);
 284             }
 285
 286           next0 = dpo0->dpoi_next_node;
 287           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 288
 289           vlib_increment_combined_counter
 290             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 291
 292           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 293                                            to_next, n_left_to_next,
 294                                            pi0, next0);
 295         }
 296
 297       vlib_put_next_frame (vm, node, next, n_left_to_next);
 298     }
 299
 300   return frame->n_vectors;
 301 }
 302
 303 /* *INDENT-OFF* */
 304 VLIB_REGISTER_NODE (ip4_load_balance_node) =
 305 {
 306   .function = ip4_load_balance,
 307   .name = "ip4-load-balance",
 308   .vector_size = sizeof (u32),
 309   .sibling_of = "ip4-lookup",
 310   .format_trace =
 311   format_ip4_lookup_trace,
 312 };
 313 /* *INDENT-ON* */
 314
 315 VLIB_NODE_FUNCTION_MULTIARCH (ip4_load_balance_node, ip4_load_balance);
 316
 317 /* get first interface address */
 318 ip4_address_t *
 319 ip4_interface_first_address (ip4_main_t * im, u32 sw_if_index,
 320                              ip_interface_address_t ** result_ia)
 321 {
 322   ip_lookup_main_t *lm = &im->lookup_main;
 323   ip_interface_address_t *ia = 0;
 324   ip4_address_t *result = 0;
 325
 326   /* *INDENT-OFF* */
 327   foreach_ip_interface_address
 328     (lm, ia, sw_if_index,
 329      1 /* honor unnumbered */ ,
 330      ({
 331        ip4_address_t * a =
 332          ip_interface_address_get_address (lm, ia);
 333        result = a;
 334        break;
 335      }));
 336   /* *INDENT-OFF* */
 337   if (result_ia)
 338     *result_ia = result ? ia : 0;
 339   return result;
 340 }
 341
 342 static void
 343 ip4_add_interface_routes (u32 sw_if_index,
 344                           ip4_main_t * im, u32 fib_index,
 345                           ip_interface_address_t * a)
 346 {
 347   ip_lookup_main_t *lm = &im->lookup_main;
 348   ip4_address_t *address = ip_interface_address_get_address (lm, a);
 349   fib_prefix_t pfx = {
 350     .fp_len = a->address_length,
 351     .fp_proto = FIB_PROTOCOL_IP4,
 352     .fp_addr.ip4 = *address,
 353   };
 354
 355   if (pfx.fp_len <= 30)
 356     {
 357       /* a /30 or shorter - add a glean for the network address */
 358       fib_table_entry_update_one_path (fib_index, &pfx,
 359                                        FIB_SOURCE_INTERFACE,
 360                                        (FIB_ENTRY_FLAG_CONNECTED |
 361                                         FIB_ENTRY_FLAG_ATTACHED),
 362                                        DPO_PROTO_IP4,
 363                                        /* No next-hop address */
 364                                        NULL,
 365                                        sw_if_index,
 366                                        // invalid FIB index
 367                                        ~0,
 368                                        1,
 369                                        // no out-label stack
 370                                        NULL,
 371                                        FIB_ROUTE_PATH_FLAG_NONE);
 372
 373       /* Add the two broadcast addresses as drop */
 374       fib_prefix_t net_pfx = {
 375         .fp_len = 32,
 376         .fp_proto = FIB_PROTOCOL_IP4,
 377         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 378       };
 379       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 380         fib_table_entry_special_add(fib_index,
 381                                     &net_pfx,
 382                                     FIB_SOURCE_INTERFACE,
 383                                     (FIB_ENTRY_FLAG_DROP |
 384                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 385       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 386       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 387         fib_table_entry_special_add(fib_index,
 388                                     &net_pfx,
 389                                     FIB_SOURCE_INTERFACE,
 390                                     (FIB_ENTRY_FLAG_DROP |
 391                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 392     }
 393   else if (pfx.fp_len == 31)
 394     {
 395       u32 mask = clib_host_to_net_u32(1);
 396       fib_prefix_t net_pfx = pfx;
 397
 398       net_pfx.fp_len = 32;
 399       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 400
 401       /* a /31 - add the other end as an attached host */
 402       fib_table_entry_update_one_path (fib_index, &net_pfx,
 403                                        FIB_SOURCE_INTERFACE,
 404                                        (FIB_ENTRY_FLAG_ATTACHED),
 405                                        DPO_PROTO_IP4,
 406                                        &net_pfx.fp_addr,
 407                                        sw_if_index,
 408                                        // invalid FIB index
 409                                        ~0,
 410                                        1,
 411                                        NULL,
 412                                        FIB_ROUTE_PATH_FLAG_NONE);
 413     }
 414   pfx.fp_len = 32;
 415
 416   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 417     {
 418       u32 classify_table_index =
 419         lm->classify_table_index_by_sw_if_index[sw_if_index];
 420       if (classify_table_index != (u32) ~ 0)
 421         {
 422           dpo_id_t dpo = DPO_INVALID;
 423
 424           dpo_set (&dpo,
 425                    DPO_CLASSIFY,
 426                    DPO_PROTO_IP4,
 427                    classify_dpo_create (DPO_PROTO_IP4, classify_table_index));
 428
 429           fib_table_entry_special_dpo_add (fib_index,
 430                                            &pfx,
 431                                            FIB_SOURCE_CLASSIFY,
 432                                            FIB_ENTRY_FLAG_NONE, &dpo);
 433           dpo_reset (&dpo);
 434         }
 435     }
 436
 437   fib_table_entry_update_one_path (fib_index, &pfx,
 438                                    FIB_SOURCE_INTERFACE,
 439                                    (FIB_ENTRY_FLAG_CONNECTED |
 440                                     FIB_ENTRY_FLAG_LOCAL),
 441                                    DPO_PROTO_IP4,
 442                                    &pfx.fp_addr,
 443                                    sw_if_index,
 444                                    // invalid FIB index
 445                                    ~0,
 446                                    1, NULL,
 447                                    FIB_ROUTE_PATH_FLAG_NONE);
 448 }
 449
 450 static void
 451 ip4_del_interface_routes (ip4_main_t * im,
 452                           u32 fib_index,
 453                           ip4_address_t * address, u32 address_length)
 454 {
 455   fib_prefix_t pfx = {
 456     .fp_len = address_length,
 457     .fp_proto = FIB_PROTOCOL_IP4,
 458     .fp_addr.ip4 = *address,
 459   };
 460
 461   if (pfx.fp_len <= 30)
 462     {
 463       fib_prefix_t net_pfx = {
 464         .fp_len = 32,
 465         .fp_proto = FIB_PROTOCOL_IP4,
 466         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 467       };
 468       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 469         fib_table_entry_special_remove(fib_index,
 470                                        &net_pfx,
 471                                        FIB_SOURCE_INTERFACE);
 472       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 473       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 474         fib_table_entry_special_remove(fib_index,
 475                                        &net_pfx,
 476                                        FIB_SOURCE_INTERFACE);
 477       fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 478     }
 479     else if (pfx.fp_len == 31)
 480     {
 481       u32 mask = clib_host_to_net_u32(1);
 482       fib_prefix_t net_pfx = pfx;
 483
 484       net_pfx.fp_len = 32;
 485       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 486
 487       fib_table_entry_delete (fib_index, &net_pfx, FIB_SOURCE_INTERFACE);
 488     }
 489
 490   pfx.fp_len = 32;
 491   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 492 }
 493
 494 void
 495 ip4_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 496 {
 497   ip4_main_t *im = &ip4_main;
 498
 499   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 500
 501   /*
 502    * enable/disable only on the 1<->0 transition
 503    */
 504   if (is_enable)
 505     {
 506       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 507         return;
 508     }
 509   else
 510     {
 511       ASSERT (im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 512       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 513         return;
 514     }
 515   vnet_feature_enable_disable ("ip4-unicast", "ip4-not-enabled", sw_if_index,
 516                                !is_enable, 0, 0);
 517
 518
 519   vnet_feature_enable_disable ("ip4-multicast", "ip4-not-enabled",
 520                                sw_if_index, !is_enable, 0, 0);
 521 }
 522
 523 static clib_error_t *
 524 ip4_add_del_interface_address_internal (vlib_main_t * vm,
 525                                         u32 sw_if_index,
 526                                         ip4_address_t * address,
 527                                         u32 address_length, u32 is_del)
 528 {
 529   vnet_main_t *vnm = vnet_get_main ();
 530   ip4_main_t *im = &ip4_main;
 531   ip_lookup_main_t *lm = &im->lookup_main;
 532   clib_error_t *error = 0;
 533   u32 if_address_index, elts_before;
 534   ip4_address_fib_t ip4_af, *addr_fib = 0;
 535
 536   /* local0 interface doesn't support IP addressing  */
 537   if (sw_if_index == 0)
 538     {
 539       return
 540        clib_error_create ("local0 interface doesn't support IP addressing");
 541     }
 542
 543   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 544   ip4_addr_fib_init (&ip4_af, address,
 545                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 546   vec_add1 (addr_fib, ip4_af);
 547
 548   /* FIXME-LATER
 549    * there is no support for adj-fib handling in the presence of overlapping
 550    * subnets on interfaces. Easy fix - disallow overlapping subnets, like
 551    * most routers do.
 552    */
 553   /* *INDENT-OFF* */
 554   if (!is_del)
 555     {
 556       /* When adding an address check that it does not conflict
 557          with an existing address. */
 558       ip_interface_address_t *ia;
 559       foreach_ip_interface_address
 560         (&im->lookup_main, ia, sw_if_index,
 561          0 /* honor unnumbered */ ,
 562          ({
 563            ip4_address_t * x =
 564              ip_interface_address_get_address
 565              (&im->lookup_main, ia);
 566            if (ip4_destination_matches_route
 567                (im, address, x, ia->address_length) ||
 568                ip4_destination_matches_route (im,
 569                                               x,
 570                                               address,
 571                                               address_length))
 572              return
 573                clib_error_create
 574                ("failed to add %U which conflicts with %U for interface %U",
 575                 format_ip4_address_and_length, address,
 576                 address_length,
 577                 format_ip4_address_and_length, x,
 578                 ia->address_length,
 579                 format_vnet_sw_if_index_name, vnm,
 580                 sw_if_index);
 581          }));
 582     }
 583   /* *INDENT-ON* */
 584
 585   elts_before = pool_elts (lm->if_address_pool);
 586
 587   error = ip_interface_address_add_del
 588     (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 589   if (error)
 590     goto done;
 591
 592   ip4_sw_interface_enable_disable (sw_if_index, !is_del);
 593
 594   if (is_del)
 595     ip4_del_interface_routes (im, ip4_af.fib_index, address, address_length);
 596   else
 597     ip4_add_interface_routes (sw_if_index,
 598                               im, ip4_af.fib_index,
 599                               pool_elt_at_index
 600                               (lm->if_address_pool, if_address_index));
 601
 602   /* If pool did not grow/shrink: add duplicate address. */
 603   if (elts_before != pool_elts (lm->if_address_pool))
 604     {
 605       ip4_add_del_interface_address_callback_t *cb;
 606       vec_foreach (cb, im->add_del_interface_address_callbacks)
 607         cb->function (im, cb->function_opaque, sw_if_index,
 608                       address, address_length, if_address_index, is_del);
 609     }
 610
 611 done:
 612   vec_free (addr_fib);
 613   return error;
 614 }
 615
 616 clib_error_t *
 617 ip4_add_del_interface_address (vlib_main_t * vm,
 618                                u32 sw_if_index,
 619                                ip4_address_t * address,
 620                                u32 address_length, u32 is_del)
 621 {
 622   return ip4_add_del_interface_address_internal
 623     (vm, sw_if_index, address, address_length, is_del);
 624 }
 625
 626 /* Built-in ip4 unicast rx feature path definition */
 627 /* *INDENT-OFF* */
 628 VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
 629 {
 630   .arc_name = "ip4-unicast",
 631   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 632   .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
 633 };
 634
 635 VNET_FEATURE_INIT (ip4_flow_classify, static) =
 636 {
 637   .arc_name = "ip4-unicast",
 638   .node_name = "ip4-flow-classify",
 639   .runs_before = VNET_FEATURES ("ip4-inacl"),
 640 };
 641
 642 VNET_FEATURE_INIT (ip4_inacl, static) =
 643 {
 644   .arc_name = "ip4-unicast",
 645   .node_name = "ip4-inacl",
 646   .runs_before = VNET_FEATURES ("ip4-source-check-via-rx"),
 647 };
 648
 649 VNET_FEATURE_INIT (ip4_source_check_1, static) =
 650 {
 651   .arc_name = "ip4-unicast",
 652   .node_name = "ip4-source-check-via-rx",
 653   .runs_before = VNET_FEATURES ("ip4-source-check-via-any"),
 654 };
 655
 656 VNET_FEATURE_INIT (ip4_source_check_2, static) =
 657 {
 658   .arc_name = "ip4-unicast",
 659   .node_name = "ip4-source-check-via-any",
 660   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
 661 };
 662
 663 VNET_FEATURE_INIT (ip4_source_and_port_range_check_rx, static) =
 664 {
 665   .arc_name = "ip4-unicast",
 666   .node_name = "ip4-source-and-port-range-check-rx",
 667   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
 668 };
 669
 670 VNET_FEATURE_INIT (ip4_policer_classify, static) =
 671 {
 672   .arc_name = "ip4-unicast",
 673   .node_name = "ip4-policer-classify",
 674   .runs_before = VNET_FEATURES ("ipsec-input-ip4"),
 675 };
 676
 677 VNET_FEATURE_INIT (ip4_ipsec, static) =
 678 {
 679   .arc_name = "ip4-unicast",
 680   .node_name = "ipsec-input-ip4",
 681   .runs_before = VNET_FEATURES ("vpath-input-ip4"),
 682 };
 683
 684 VNET_FEATURE_INIT (ip4_vpath, static) =
 685 {
 686   .arc_name = "ip4-unicast",
 687   .node_name = "vpath-input-ip4",
 688   .runs_before = VNET_FEATURES ("ip4-vxlan-bypass"),
 689 };
 690
 691 VNET_FEATURE_INIT (ip4_vxlan_bypass, static) =
 692 {
 693   .arc_name = "ip4-unicast",
 694   .node_name = "ip4-vxlan-bypass",
 695   .runs_before = VNET_FEATURES ("ip4-lookup"),
 696 };
 697
 698 VNET_FEATURE_INIT (ip4_not_enabled, static) =
 699 {
 700   .arc_name = "ip4-unicast",
 701   .node_name = "ip4-not-enabled",
 702   .runs_before = VNET_FEATURES ("ip4-lookup"),
 703 };
 704
 705 VNET_FEATURE_INIT (ip4_lookup, static) =
 706 {
 707   .arc_name = "ip4-unicast",
 708   .node_name = "ip4-lookup",
 709   .runs_before = 0,     /* not before any other features */
 710 };
 711
 712 /* Built-in ip4 multicast rx feature path definition */
 713 VNET_FEATURE_ARC_INIT (ip4_multicast, static) =
 714 {
 715   .arc_name = "ip4-multicast",
 716   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 717   .arc_index_ptr = &ip4_main.lookup_main.mcast_feature_arc_index,
 718 };
 719
 720 VNET_FEATURE_INIT (ip4_vpath_mc, static) =
 721 {
 722   .arc_name = "ip4-multicast",
 723   .node_name = "vpath-input-ip4",
 724   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
 725 };
 726
 727 VNET_FEATURE_INIT (ip4_mc_not_enabled, static) =
 728 {
 729   .arc_name = "ip4-multicast",
 730   .node_name = "ip4-not-enabled",
 731   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
 732 };
 733
 734 VNET_FEATURE_INIT (ip4_lookup_mc, static) =
 735 {
 736   .arc_name = "ip4-multicast",
 737   .node_name = "ip4-mfib-forward-lookup",
 738   .runs_before = 0,     /* last feature */
 739 };
 740
 741 /* Source and port-range check ip4 tx feature path definition */
 742 VNET_FEATURE_ARC_INIT (ip4_output, static) =
 743 {
 744   .arc_name = "ip4-output",
 745   .start_nodes = VNET_FEATURES ("ip4-rewrite", "ip4-midchain", "ip4-dvr-dpo"),
 746   .arc_index_ptr = &ip4_main.lookup_main.output_feature_arc_index,
 747 };
 748
 749 VNET_FEATURE_INIT (ip4_source_and_port_range_check_tx, static) =
 750 {
 751   .arc_name = "ip4-output",
 752   .node_name = "ip4-source-and-port-range-check-tx",
 753   .runs_before = VNET_FEATURES ("ip4-outacl"),
 754 };
 755
 756 VNET_FEATURE_INIT (ip4_outacl, static) =
 757 {
 758   .arc_name = "ip4-output",
 759   .node_name = "ip4-outacl",
 760   .runs_before = VNET_FEATURES ("ipsec-output-ip4"),
 761 };
 762
 763 VNET_FEATURE_INIT (ip4_ipsec_output, static) =
 764 {
 765   .arc_name = "ip4-output",
 766   .node_name = "ipsec-output-ip4",
 767   .runs_before = VNET_FEATURES ("interface-output"),
 768 };
 769
 770 /* Built-in ip4 tx feature path definition */
 771 VNET_FEATURE_INIT (ip4_interface_output, static) =
 772 {
 773   .arc_name = "ip4-output",
 774   .node_name = "interface-output",
 775   .runs_before = 0,     /* not before any other features */
 776 };
 777 /* *INDENT-ON* */
 778
 779 static clib_error_t *
 780 ip4_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 781 {
 782   ip4_main_t *im = &ip4_main;
 783
 784   /* Fill in lookup tables with default table (0). */
 785   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 786   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 787
 788   if (!is_add)
 789     {
 790       ip4_main_t *im4 = &ip4_main;
 791       ip_lookup_main_t *lm4 = &im4->lookup_main;
 792       ip_interface_address_t *ia = 0;
 793       ip4_address_t *address;
 794       vlib_main_t *vm = vlib_get_main ();
 795
 796       vnet_sw_interface_update_unnumbered (sw_if_index, ~0, 0);
 797       /* *INDENT-OFF* */
 798       foreach_ip_interface_address (lm4, ia, sw_if_index, 0,
 799       ({
 800         address = ip_interface_address_get_address (lm4, ia);
 801         ip4_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
 802       }));
 803       /* *INDENT-ON* */
 804     }
 805
 806   vnet_feature_enable_disable ("ip4-unicast", "ip4-not-enabled", sw_if_index,
 807                                is_add, 0, 0);
 808
 809   vnet_feature_enable_disable ("ip4-multicast", "ip4-not-enabled",
 810                                sw_if_index, is_add, 0, 0);
 811
 812   return /* no error */ 0;
 813 }
 814
 815 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
 816
 817 /* Global IP4 main. */
 818 ip4_main_t ip4_main;
 819
 820 clib_error_t *
 821 ip4_lookup_init (vlib_main_t * vm)
 822 {
 823   ip4_main_t *im = &ip4_main;
 824   clib_error_t *error;
 825   uword i;
 826
 827   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
 828     return error;
 829   if ((error = vlib_call_init_function (vm, ip4_mtrie_module_init)))
 830     return (error);
 831   if ((error = vlib_call_init_function (vm, fib_module_init)))
 832     return error;
 833   if ((error = vlib_call_init_function (vm, mfib_module_init)))
 834     return error;
 835
 836   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
 837     {
 838       u32 m;
 839
 840       if (i < 32)
 841         m = pow2_mask (i) << (32 - i);
 842       else
 843         m = ~0;
 844       im->fib_masks[i] = clib_host_to_net_u32 (m);
 845     }
 846
 847   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
 848
 849   /* Create FIB with index 0 and table id of 0. */
 850   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0,
 851                                      FIB_SOURCE_DEFAULT_ROUTE);
 852   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0,
 853                                       MFIB_SOURCE_DEFAULT_ROUTE);
 854
 855   {
 856     pg_node_t *pn;
 857     pn = pg_get_node (ip4_lookup_node.index);
 858     pn->unformat_edit = unformat_pg_ip4_header;
 859   }
 860
 861   {
 862     ethernet_arp_header_t h;
 863
 864     memset (&h, 0, sizeof (h));
 865
 866     /* Set target ethernet address to all zeros. */
 867     memset (h.ip4_over_ethernet[1].ethernet, 0,
 868             sizeof (h.ip4_over_ethernet[1].ethernet));
 869
 870 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
 871 #define _8(f,v) h.f = v;
 872     _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
 873     _16 (l3_type, ETHERNET_TYPE_IP4);
 874     _8 (n_l2_address_bytes, 6);
 875     _8 (n_l3_address_bytes, 4);
 876     _16 (opcode, ETHERNET_ARP_OPCODE_request);
 877 #undef _16
 878 #undef _8
 879
 880     vlib_packet_template_init (vm, &im->ip4_arp_request_packet_template,
 881                                /* data */ &h,
 882                                sizeof (h),
 883                                /* alloc chunk size */ 8,
 884                                "ip4 arp");
 885   }
 886
 887   return error;
 888 }
 889
 890 VLIB_INIT_FUNCTION (ip4_lookup_init);
 891
 892 typedef struct
 893 {
 894   /* Adjacency taken. */
 895   u32 dpo_index;
 896   u32 flow_hash;
 897   u32 fib_index;
 898
 899   /* Packet data, possibly *after* rewrite. */
 900   u8 packet_data[64 - 1 * sizeof (u32)];
 901 }
 902 ip4_forward_next_trace_t;
 903
 904 u8 *
 905 format_ip4_forward_next_trace (u8 * s, va_list * args)
 906 {
 907   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 908   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 909   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
 910   u32 indent = format_get_indent (s);
 911   s = format (s, "%U%U",
 912               format_white_space, indent,
 913               format_ip4_header, t->packet_data, sizeof (t->packet_data));
 914   return s;
 915 }
 916
 917 static u8 *
 918 format_ip4_lookup_trace (u8 * s, va_list * args)
 919 {
 920   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 921   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 922   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
 923   u32 indent = format_get_indent (s);
 924
 925   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 926               t->fib_index, t->dpo_index, t->flow_hash);
 927   s = format (s, "\n%U%U",
 928               format_white_space, indent,
 929               format_ip4_header, t->packet_data, sizeof (t->packet_data));
 930   return s;
 931 }
 932
 933 static u8 *
 934 format_ip4_rewrite_trace (u8 * s, va_list * args)
 935 {
 936   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 937   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 938   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
 939   u32 indent = format_get_indent (s);
 940
 941   s = format (s, "tx_sw_if_index %d dpo-idx %d : %U flow hash: 0x%08x",
 942               t->fib_index, t->dpo_index, format_ip_adjacency,
 943               t->dpo_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
 944   s = format (s, "\n%U%U",
 945               format_white_space, indent,
 946               format_ip_adjacency_packet_data,
 947               t->dpo_index, t->packet_data, sizeof (t->packet_data));
 948   return s;
 949 }
 950
 951 /* Common trace function for all ip4-forward next nodes. */
 952 void
 953 ip4_forward_next_trace (vlib_main_t * vm,
 954                         vlib_node_runtime_t * node,
 955                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
 956 {
 957   u32 *from, n_left;
 958   ip4_main_t *im = &ip4_main;
 959
 960   n_left = frame->n_vectors;
 961   from = vlib_frame_vector_args (frame);
 962
 963   while (n_left >= 4)
 964     {
 965       u32 bi0, bi1;
 966       vlib_buffer_t *b0, *b1;
 967       ip4_forward_next_trace_t *t0, *t1;
 968
 969       /* Prefetch next iteration. */
 970       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
 971       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
 972
 973       bi0 = from[0];
 974       bi1 = from[1];
 975
 976       b0 = vlib_get_buffer (vm, bi0);
 977       b1 = vlib_get_buffer (vm, bi1);
 978
 979       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 980         {
 981           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 982           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
 983           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
 984           t0->fib_index =
 985             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
 986              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
 987             vec_elt (im->fib_index_by_sw_if_index,
 988                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
 989
 990           clib_memcpy (t0->packet_data,
 991                        vlib_buffer_get_current (b0),
 992                        sizeof (t0->packet_data));
 993         }
 994       if (b1->flags & VLIB_BUFFER_IS_TRACED)
 995         {
 996           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
 997           t1->dpo_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
 998           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
 999           t1->fib_index =
1000             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
1001              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
1002             vec_elt (im->fib_index_by_sw_if_index,
1003                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
1004           clib_memcpy (t1->packet_data, vlib_buffer_get_current (b1),
1005                        sizeof (t1->packet_data));
1006         }
1007       from += 2;
1008       n_left -= 2;
1009     }
1010
1011   while (n_left >= 1)
1012     {
1013       u32 bi0;
1014       vlib_buffer_t *b0;
1015       ip4_forward_next_trace_t *t0;
1016
1017       bi0 = from[0];
1018
1019       b0 = vlib_get_buffer (vm, bi0);
1020
1021       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1022         {
1023           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1024           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1025           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1026           t0->fib_index =
1027             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1028              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1029             vec_elt (im->fib_index_by_sw_if_index,
1030                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1031           clib_memcpy (t0->packet_data, vlib_buffer_get_current (b0),
1032                        sizeof (t0->packet_data));
1033         }
1034       from += 1;
1035       n_left -= 1;
1036     }
1037 }
1038
1039 /* Compute TCP/UDP/ICMP4 checksum in software. */
1040 u16
1041 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1042                               ip4_header_t * ip0)
1043 {
1044   ip_csum_t sum0;
1045   u32 ip_header_length, payload_length_host_byte_order;
1046   u32 n_this_buffer, n_bytes_left, n_ip_bytes_this_buffer;
1047   u16 sum16;
1048   void *data_this_buffer;
1049
1050   /* Initialize checksum with ip header. */
1051   ip_header_length = ip4_header_bytes (ip0);
1052   payload_length_host_byte_order =
1053     clib_net_to_host_u16 (ip0->length) - ip_header_length;
1054   sum0 =
1055     clib_host_to_net_u32 (payload_length_host_byte_order +
1056                           (ip0->protocol << 16));
1057
1058   if (BITS (uword) == 32)
1059     {
1060       sum0 =
1061         ip_csum_with_carry (sum0,
1062                             clib_mem_unaligned (&ip0->src_address, u32));
1063       sum0 =
1064         ip_csum_with_carry (sum0,
1065                             clib_mem_unaligned (&ip0->dst_address, u32));
1066     }
1067   else
1068     sum0 =
1069       ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1070
1071   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1072   data_this_buffer = (void *) ip0 + ip_header_length;
1073   n_ip_bytes_this_buffer =
1074     p0->current_length - (((u8 *) ip0 - p0->data) - p0->current_data);
1075   if (n_this_buffer + ip_header_length > n_ip_bytes_this_buffer)
1076     {
1077       n_this_buffer = n_ip_bytes_this_buffer > ip_header_length ?
1078         n_ip_bytes_this_buffer - ip_header_length : 0;
1079     }
1080   while (1)
1081     {
1082       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1083       n_bytes_left -= n_this_buffer;
1084       if (n_bytes_left == 0)
1085         break;
1086
1087       ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1088       p0 = vlib_get_buffer (vm, p0->next_buffer);
1089       data_this_buffer = vlib_buffer_get_current (p0);
1090       n_this_buffer = p0->current_length;
1091     }
1092
1093   sum16 = ~ip_csum_fold (sum0);
1094
1095   return sum16;
1096 }
1097
1098 u32
1099 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1100 {
1101   ip4_header_t *ip0 = vlib_buffer_get_current (p0);
1102   udp_header_t *udp0;
1103   u16 sum16;
1104
1105   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1106           || ip0->protocol == IP_PROTOCOL_UDP);
1107
1108   udp0 = (void *) (ip0 + 1);
1109   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1110     {
1111       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1112                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
1113       return p0->flags;
1114     }
1115
1116   sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1117
1118   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1119                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
1120
1121   return p0->flags;
1122 }
1123
1124 /* *INDENT-OFF* */
1125 VNET_FEATURE_ARC_INIT (ip4_local) =
1126 {
1127   .arc_name  = "ip4-local",
1128   .start_nodes = VNET_FEATURES ("ip4-local"),
1129 };
1130 /* *INDENT-ON* */
1131
1132 static inline void
1133 ip4_local_validate_l4 (vlib_main_t * vm, vlib_buffer_t * p, ip4_header_t * ip,
1134                        u8 is_udp, u8 * error, u8 * good_tcp_udp)
1135 {
1136   u32 flags0;
1137   flags0 = ip4_tcp_udp_validate_checksum (vm, p);
1138   *good_tcp_udp = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1139   if (is_udp)
1140     {
1141       udp_header_t *udp;
1142       u32 ip_len, udp_len;
1143       i32 len_diff;
1144       udp = ip4_next_header (ip);
1145       /* Verify UDP length. */
1146       ip_len = clib_net_to_host_u16 (ip->length);
1147       udp_len = clib_net_to_host_u16 (udp->length);
1148
1149       len_diff = ip_len - udp_len;
1150       *good_tcp_udp &= len_diff >= 0;
1151       *error = len_diff < 0 ? IP4_ERROR_UDP_LENGTH : *error;
1152     }
1153 }
1154
1155 #define ip4_local_do_l4_check(is_tcp_udp, flags)                        \
1156     (is_tcp_udp && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED \
1157     || flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM \
1158     || flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
1159
1160 static inline uword
1161 ip4_local_inline (vlib_main_t * vm,
1162                   vlib_node_runtime_t * node,
1163                   vlib_frame_t * frame, int head_of_feature_arc)
1164 {
1165   ip4_main_t *im = &ip4_main;
1166   ip_lookup_main_t *lm = &im->lookup_main;
1167   ip_local_next_t next_index;
1168   u32 *from, *to_next, n_left_from, n_left_to_next;
1169   vlib_node_runtime_t *error_node =
1170     vlib_node_get_runtime (vm, ip4_input_node.index);
1171   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1172
1173   from = vlib_frame_vector_args (frame);
1174   n_left_from = frame->n_vectors;
1175   next_index = node->cached_next_index;
1176
1177   if (node->flags & VLIB_NODE_FLAG_TRACE)
1178     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1179
1180   while (n_left_from > 0)
1181     {
1182       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1183
1184       while (n_left_from >= 4 && n_left_to_next >= 2)
1185         {
1186           vlib_buffer_t *p0, *p1;
1187           ip4_header_t *ip0, *ip1;
1188           ip4_fib_mtrie_t *mtrie0, *mtrie1;
1189           ip4_fib_mtrie_leaf_t leaf0, leaf1;
1190           const dpo_id_t *dpo0, *dpo1;
1191           const load_balance_t *lb0, *lb1;
1192           u32 pi0, next0, fib_index0, lbi0;
1193           u32 pi1, next1, fib_index1, lbi1;
1194           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1195           u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1196           u32 sw_if_index0, sw_if_index1;
1197
1198           pi0 = to_next[0] = from[0];
1199           pi1 = to_next[1] = from[1];
1200           from += 2;
1201           n_left_from -= 2;
1202           to_next += 2;
1203           n_left_to_next -= 2;
1204
1205           next0 = next1 = IP_LOCAL_NEXT_DROP;
1206           error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1207
1208           p0 = vlib_get_buffer (vm, pi0);
1209           p1 = vlib_get_buffer (vm, pi1);
1210
1211           ip0 = vlib_buffer_get_current (p0);
1212           ip1 = vlib_buffer_get_current (p1);
1213
1214           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1215           vnet_buffer (p1)->l3_hdr_offset = p1->current_data;
1216
1217           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1218           sw_if_index1 = vnet_buffer (p1)->sw_if_index[VLIB_RX];
1219
1220           /* Treat IP frag packets as "experimental" protocol for now
1221              until support of IP frag reassembly is implemented */
1222           proto0 =
1223             ip4_is_fragment (ip0) ? IP_PROTOCOL_VPP_FRAGMENTATION :
1224             ip0->protocol;
1225           proto1 =
1226             ip4_is_fragment (ip1) ? IP_PROTOCOL_VPP_FRAGMENTATION :
1227             ip1->protocol;
1228
1229           if (head_of_feature_arc == 0)
1230             goto skip_checks;
1231
1232           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1233           is_udp1 = proto1 == IP_PROTOCOL_UDP;
1234           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1235           is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1236
1237           good_tcp_udp0 =
1238             (p0->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1239              || (p0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1240                  || p0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) != 0;
1241           good_tcp_udp1 = (p1->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1242                            || (p1->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1243                                || p1->flags &
1244                                VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) != 0;
1245
1246           if (PREDICT_FALSE (ip4_local_do_l4_check (is_tcp_udp0, p0->flags)
1247                              || ip4_local_do_l4_check (is_tcp_udp1,
1248                                                        p1->flags)))
1249             {
1250               if (is_tcp_udp0)
1251                 ip4_local_validate_l4 (vm, p0, ip0, is_udp0, &error0,
1252                                        &good_tcp_udp0);
1253               if (is_tcp_udp1)
1254                 ip4_local_validate_l4 (vm, p1, ip1, is_udp1, &error1,
1255                                        &good_tcp_udp1);
1256             }
1257
1258           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1259           error0 = (is_tcp_udp0 && !good_tcp_udp0
1260                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1261           error1 = (is_tcp_udp1 && !good_tcp_udp1
1262                     ? IP4_ERROR_TCP_CHECKSUM + is_udp1 : error1);
1263
1264           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1265           fib_index0 =
1266             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1267              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1268
1269           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index1);
1270           fib_index1 =
1271             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
1272              (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
1273
1274           /* TODO maybe move to lookup? */
1275           vnet_buffer (p0)->ip.fib_index = fib_index0;
1276           vnet_buffer (p1)->ip.fib_index = fib_index1;
1277
1278           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1279           mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
1280
1281           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1282           leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, &ip1->src_address);
1283           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1284                                              2);
1285           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address,
1286                                              2);
1287           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1288                                              3);
1289           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address,
1290                                              3);
1291
1292           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0 =
1293             ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1294           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1295
1296           vnet_buffer (p1)->ip.adj_index[VLIB_RX] = lbi1 =
1297             ip4_fib_mtrie_leaf_get_adj_index (leaf1);
1298           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = lbi1;
1299
1300           lb0 = load_balance_get (lbi0);
1301           lb1 = load_balance_get (lbi1);
1302           dpo0 = load_balance_get_bucket_i (lb0, 0);
1303           dpo1 = load_balance_get_bucket_i (lb1, 0);
1304
1305           /*
1306            * Must have a route to source otherwise we drop the packet.
1307            * ip4 broadcasts are accepted, e.g. to make dhcp client work
1308            *
1309            * The checks are:
1310            *  - the source is a recieve => it's from us => bogus, do this
1311            *    first since it sets a different error code.
1312            *  - uRPF check for any route to source - accept if passes.
1313            *  - allow packets destined to the broadcast address from unknown sources
1314            */
1315           if (p0->flags & VNET_BUFFER_F_IS_NATED)
1316             goto skip_check0;
1317
1318           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1319                      dpo0->dpoi_type == DPO_RECEIVE) ?
1320                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1321           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1322                      !fib_urpf_check_size (lb0->lb_urpf) &&
1323                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1324                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1325
1326         skip_check0:
1327           if (p1->flags & VNET_BUFFER_F_IS_NATED)
1328             goto skip_checks;
1329
1330           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1331                      dpo1->dpoi_type == DPO_RECEIVE) ?
1332                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error1);
1333           error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1334                      !fib_urpf_check_size (lb1->lb_urpf) &&
1335                      ip1->dst_address.as_u32 != 0xFFFFFFFF)
1336                     ? IP4_ERROR_SRC_LOOKUP_MISS : error1);
1337
1338         skip_checks:
1339
1340           next0 = lm->local_next_by_ip_protocol[proto0];
1341           next1 = lm->local_next_by_ip_protocol[proto1];
1342
1343           next0 =
1344             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1345           next1 =
1346             error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1347
1348           p0->error = error0 ? error_node->errors[error0] : 0;
1349           p1->error = error1 ? error_node->errors[error1] : 0;
1350
1351           if (head_of_feature_arc)
1352             {
1353               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1354                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1355               if (PREDICT_TRUE (error1 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1356                 vnet_feature_arc_start (arc_index, sw_if_index1, &next1, p1);
1357             }
1358
1359           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1360                                            n_left_to_next, pi0, pi1,
1361                                            next0, next1);
1362         }
1363
1364       while (n_left_from > 0 && n_left_to_next > 0)
1365         {
1366           vlib_buffer_t *p0;
1367           ip4_header_t *ip0;
1368           ip4_fib_mtrie_t *mtrie0;
1369           ip4_fib_mtrie_leaf_t leaf0;
1370           u32 pi0, next0, fib_index0, lbi0;
1371           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1372           load_balance_t *lb0;
1373           const dpo_id_t *dpo0;
1374           u32 sw_if_index0;
1375
1376           pi0 = to_next[0] = from[0];
1377           from += 1;
1378           n_left_from -= 1;
1379           to_next += 1;
1380           n_left_to_next -= 1;
1381
1382           next0 = IP_LOCAL_NEXT_DROP;
1383           error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
1384
1385           p0 = vlib_get_buffer (vm, pi0);
1386           ip0 = vlib_buffer_get_current (p0);
1387           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1388           sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
1389
1390           /* Treat IP frag packets as "experimental" protocol for now
1391              until support of IP frag reassembly is implemented */
1392           proto0 =
1393             ip4_is_fragment (ip0) ? IP_PROTOCOL_VPP_FRAGMENTATION :
1394             ip0->protocol;
1395
1396           if (head_of_feature_arc == 0 || p0->flags & VNET_BUFFER_F_IS_NATED)
1397             goto skip_check;
1398
1399           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1400           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1401
1402           good_tcp_udp0 =
1403             (p0->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1404              || (p0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1405                  || p0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) != 0;
1406
1407           if (PREDICT_FALSE (ip4_local_do_l4_check (is_tcp_udp0, p0->flags)))
1408             {
1409               ip4_local_validate_l4 (vm, p0, ip0, is_udp0, &error0,
1410                                      &good_tcp_udp0);
1411             }
1412
1413           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1414           error0 = (is_tcp_udp0 && !good_tcp_udp0
1415                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1416
1417           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, sw_if_index0);
1418           fib_index0 =
1419             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1420              (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1421           vnet_buffer (p0)->ip.fib_index = fib_index0;
1422           mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1423           leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1424           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1425                                              2);
1426           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address,
1427                                              3);
1428           lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1429           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1430           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0;
1431
1432           lb0 = load_balance_get (lbi0);
1433           dpo0 = load_balance_get_bucket_i (lb0, 0);
1434
1435           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1436                      dpo0->dpoi_type == DPO_RECEIVE) ?
1437                     IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1438           error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1439                      !fib_urpf_check_size (lb0->lb_urpf) &&
1440                      ip0->dst_address.as_u32 != 0xFFFFFFFF)
1441                     ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1442
1443         skip_check:
1444           next0 = lm->local_next_by_ip_protocol[proto0];
1445           next0 =
1446             error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1447
1448           p0->error = error0 ? error_node->errors[error0] : 0;
1449
1450           if (head_of_feature_arc)
1451             {
1452               if (PREDICT_TRUE (error0 == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1453                 vnet_feature_arc_start (arc_index, sw_if_index0, &next0, p0);
1454             }
1455
1456           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1457                                            n_left_to_next, pi0, next0);
1458         }
1459       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1460     }
1461
1462   return frame->n_vectors;
1463 }
1464
1465 static uword
1466 ip4_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1467 {
1468   return ip4_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1469 }
1470
1471 /* *INDENT-OFF* */
1472 VLIB_REGISTER_NODE (ip4_local_node) =
1473 {
1474   .function = ip4_local,
1475   .name = "ip4-local",
1476   .vector_size = sizeof (u32),
1477   .format_trace = format_ip4_forward_next_trace,
1478   .n_next_nodes = IP_LOCAL_N_NEXT,
1479   .next_nodes =
1480   {
1481     [IP_LOCAL_NEXT_DROP] = "ip4-drop",
1482     [IP_LOCAL_NEXT_PUNT] = "ip4-punt",
1483     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
1484     [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",
1485     [IP_LOCAL_NEXT_REASSEMBLY] = "ip4-reassembly",
1486   },
1487 };
1488 /* *INDENT-ON* */
1489
1490 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local);
1491
1492 static uword
1493 ip4_local_end_of_arc (vlib_main_t * vm,
1494                       vlib_node_runtime_t * node, vlib_frame_t * frame)
1495 {
1496   return ip4_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1497 }
1498
1499 /* *INDENT-OFF* */
1500 VLIB_REGISTER_NODE (ip4_local_end_of_arc_node,static) = {
1501   .function = ip4_local_end_of_arc,
1502   .name = "ip4-local-end-of-arc",
1503   .vector_size = sizeof (u32),
1504
1505   .format_trace = format_ip4_forward_next_trace,
1506   .sibling_of = "ip4-local",
1507 };
1508
1509 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_end_of_arc_node, ip4_local_end_of_arc)
1510
1511 VNET_FEATURE_INIT (ip4_local_end_of_arc, static) = {
1512   .arc_name = "ip4-local",
1513   .node_name = "ip4-local-end-of-arc",
1514   .runs_before = 0, /* not before any other features */
1515 };
1516 /* *INDENT-ON* */
1517
1518 void
1519 ip4_register_protocol (u32 protocol, u32 node_index)
1520 {
1521   vlib_main_t *vm = vlib_get_main ();
1522   ip4_main_t *im = &ip4_main;
1523   ip_lookup_main_t *lm = &im->lookup_main;
1524
1525   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1526   lm->local_next_by_ip_protocol[protocol] =
1527     vlib_node_add_next (vm, ip4_local_node.index, node_index);
1528 }
1529
1530 static clib_error_t *
1531 show_ip_local_command_fn (vlib_main_t * vm,
1532                           unformat_input_t * input, vlib_cli_command_t * cmd)
1533 {
1534   ip4_main_t *im = &ip4_main;
1535   ip_lookup_main_t *lm = &im->lookup_main;
1536   int i;
1537
1538   vlib_cli_output (vm, "Protocols handled by ip4_local");
1539   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
1540     {
1541       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
1542         {
1543           u32 node_index = vlib_get_node (vm,
1544                                           ip4_local_node.index)->
1545             next_nodes[lm->local_next_by_ip_protocol[i]];
1546           vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm,
1547                            node_index);
1548         }
1549     }
1550   return 0;
1551 }
1552
1553
1554
1555 /*?
1556  * Display the set of protocols handled by the local IPv4 stack.
1557  *
1558  * @cliexpar
1559  * Example of how to display local protocol table:
1560  * @cliexstart{show ip local}
1561  * Protocols handled by ip4_local
1562  * 1
1563  * 17
1564  * 47
1565  * @cliexend
1566 ?*/
1567 /* *INDENT-OFF* */
1568 VLIB_CLI_COMMAND (show_ip_local, static) =
1569 {
1570   .path = "show ip local",
1571   .function = show_ip_local_command_fn,
1572   .short_help = "show ip local",
1573 };
1574 /* *INDENT-ON* */
1575
1576 always_inline uword
1577 ip4_arp_inline (vlib_main_t * vm,
1578                 vlib_node_runtime_t * node,
1579                 vlib_frame_t * frame, int is_glean)
1580 {
1581   vnet_main_t *vnm = vnet_get_main ();
1582   ip4_main_t *im = &ip4_main;
1583   ip_lookup_main_t *lm = &im->lookup_main;
1584   u32 *from, *to_next_drop;
1585   uword n_left_from, n_left_to_next_drop, next_index;
1586   static f64 time_last_seed_change = -1e100;
1587   static u32 hash_seeds[3];
1588   static uword hash_bitmap[256 / BITS (uword)];
1589   f64 time_now;
1590
1591   if (node->flags & VLIB_NODE_FLAG_TRACE)
1592     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1593
1594   time_now = vlib_time_now (vm);
1595   if (time_now - time_last_seed_change > 1e-3)
1596     {
1597       uword i;
1598       u32 *r = clib_random_buffer_get_data (&vm->random_buffer,
1599                                             sizeof (hash_seeds));
1600       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1601         hash_seeds[i] = r[i];
1602
1603       /* Mark all hash keys as been no-seen before. */
1604       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1605         hash_bitmap[i] = 0;
1606
1607       time_last_seed_change = time_now;
1608     }
1609
1610   from = vlib_frame_vector_args (frame);
1611   n_left_from = frame->n_vectors;
1612   next_index = node->cached_next_index;
1613   if (next_index == IP4_ARP_NEXT_DROP)
1614     next_index = IP4_ARP_N_NEXT;        /* point to first interface */
1615
1616   while (n_left_from > 0)
1617     {
1618       vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
1619                            to_next_drop, n_left_to_next_drop);
1620
1621       while (n_left_from > 0 && n_left_to_next_drop > 0)
1622         {
1623           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
1624           ip_adjacency_t *adj0;
1625           vlib_buffer_t *p0;
1626           ip4_header_t *ip0;
1627           uword bm0;
1628
1629           pi0 = from[0];
1630
1631           p0 = vlib_get_buffer (vm, pi0);
1632
1633           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1634           adj0 = adj_get (adj_index0);
1635           ip0 = vlib_buffer_get_current (p0);
1636
1637           a0 = hash_seeds[0];
1638           b0 = hash_seeds[1];
1639           c0 = hash_seeds[2];
1640
1641           sw_if_index0 = adj0->rewrite_header.sw_if_index;
1642           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
1643
1644           if (is_glean)
1645             {
1646               /*
1647                * this is the Glean case, so we are ARPing for the
1648                * packet's destination
1649                */
1650               a0 ^= ip0->dst_address.data_u32;
1651             }
1652           else
1653             {
1654               a0 ^= adj0->sub_type.nbr.next_hop.ip4.data_u32;
1655             }
1656           b0 ^= sw_if_index0;
1657
1658           hash_v3_mix32 (a0, b0, c0);
1659           hash_v3_finalize32 (a0, b0, c0);
1660
1661           c0 &= BITS (hash_bitmap) - 1;
1662           m0 = (uword) 1 << (c0 % BITS (uword));
1663           c0 = c0 / BITS (uword);
1664
1665           bm0 = hash_bitmap[c0];
1666           drop0 = (bm0 & m0) != 0;
1667
1668           /* Mark it as seen. */
1669           hash_bitmap[c0] = bm0 | m0;
1670
1671           from += 1;
1672           n_left_from -= 1;
1673           to_next_drop[0] = pi0;
1674           to_next_drop += 1;
1675           n_left_to_next_drop -= 1;
1676
1677           p0->error =
1678             node->errors[drop0 ? IP4_ARP_ERROR_DROP :
1679                          IP4_ARP_ERROR_REQUEST_SENT];
1680
1681           /*
1682            * the adj has been updated to a rewrite but the node the DPO that got
1683            * us here hasn't - yet. no big deal. we'll drop while we wait.
1684            */
1685           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
1686             continue;
1687
1688           if (drop0)
1689             continue;
1690
1691           /*
1692            * Can happen if the control-plane is programming tables
1693            * with traffic flowing; at least that's today's lame excuse.
1694            */
1695           if ((is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_GLEAN)
1696               || (!is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP))
1697             {
1698               p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
1699             }
1700           else
1701             /* Send ARP request. */
1702             {
1703               u32 bi0 = 0;
1704               vlib_buffer_t *b0;
1705               ethernet_arp_header_t *h0;
1706               vnet_hw_interface_t *hw_if0;
1707
1708               h0 =
1709                 vlib_packet_template_get_packet (vm,
1710                                                  &im->ip4_arp_request_packet_template,
1711                                                  &bi0);
1712
1713               /* Seems we're out of buffers */
1714               if (PREDICT_FALSE (!h0))
1715                 continue;
1716
1717               /* Add rewrite/encap string for ARP packet. */
1718               vnet_rewrite_one_header (adj0[0], h0,
1719                                        sizeof (ethernet_header_t));
1720
1721               hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1722
1723               /* Src ethernet address in ARP header. */
1724               clib_memcpy (h0->ip4_over_ethernet[0].ethernet,
1725                            hw_if0->hw_address,
1726                            sizeof (h0->ip4_over_ethernet[0].ethernet));
1727
1728               if (is_glean)
1729                 {
1730                   /* The interface's source address is stashed in the Glean Adj */
1731                   h0->ip4_over_ethernet[0].ip4 =
1732                     adj0->sub_type.glean.receive_addr.ip4;
1733
1734                   /* Copy in destination address we are requesting. This is the
1735                    * glean case, so it's the packet's destination.*/
1736                   h0->ip4_over_ethernet[1].ip4.data_u32 =
1737                     ip0->dst_address.data_u32;
1738                 }
1739               else
1740                 {
1741                   /* Src IP address in ARP header. */
1742                   if (ip4_src_address_for_packet (lm, sw_if_index0,
1743                                                   &h0->
1744                                                   ip4_over_ethernet[0].ip4))
1745                     {
1746                       /* No source address available */
1747                       p0->error =
1748                         node->errors[IP4_ARP_ERROR_NO_SOURCE_ADDRESS];
1749                       vlib_buffer_free (vm, &bi0, 1);
1750                       continue;
1751                     }
1752
1753                   /* Copy in destination address we are requesting from the
1754                      incomplete adj */
1755                   h0->ip4_over_ethernet[1].ip4.data_u32 =
1756                     adj0->sub_type.nbr.next_hop.ip4.as_u32;
1757                 }
1758
1759               vlib_buffer_copy_trace_flag (vm, p0, bi0);
1760               b0 = vlib_get_buffer (vm, bi0);
1761               VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
1762               vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
1763
1764               vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
1765
1766               vlib_set_next_frame_buffer (vm, node,
1767                                           adj0->rewrite_header.next_index,
1768                                           bi0);
1769             }
1770         }
1771
1772       vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
1773     }
1774
1775   return frame->n_vectors;
1776 }
1777
1778 static uword
1779 ip4_arp (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1780 {
1781   return (ip4_arp_inline (vm, node, frame, 0));
1782 }
1783
1784 static uword
1785 ip4_glean (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1786 {
1787   return (ip4_arp_inline (vm, node, frame, 1));
1788 }
1789
1790 static char *ip4_arp_error_strings[] = {
1791   [IP4_ARP_ERROR_DROP] = "address overflow drops",
1792   [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
1793   [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
1794   [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
1795   [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
1796   [IP4_ARP_ERROR_NO_SOURCE_ADDRESS] = "no source address for ARP request",
1797 };
1798
1799 /* *INDENT-OFF* */
1800 VLIB_REGISTER_NODE (ip4_arp_node) =
1801 {
1802   .function = ip4_arp,
1803   .name = "ip4-arp",
1804   .vector_size = sizeof (u32),
1805   .format_trace = format_ip4_forward_next_trace,
1806   .n_errors = ARRAY_LEN (ip4_arp_error_strings),
1807   .error_strings = ip4_arp_error_strings,
1808   .n_next_nodes = IP4_ARP_N_NEXT,
1809   .next_nodes =
1810   {
1811     [IP4_ARP_NEXT_DROP] = "error-drop",
1812   },
1813 };
1814
1815 VLIB_REGISTER_NODE (ip4_glean_node) =
1816 {
1817   .function = ip4_glean,
1818   .name = "ip4-glean",
1819   .vector_size = sizeof (u32),
1820   .format_trace = format_ip4_forward_next_trace,
1821   .n_errors = ARRAY_LEN (ip4_arp_error_strings),
1822   .error_strings = ip4_arp_error_strings,
1823   .n_next_nodes = IP4_ARP_N_NEXT,
1824   .next_nodes = {
1825   [IP4_ARP_NEXT_DROP] = "error-drop",
1826   },
1827 };
1828 /* *INDENT-ON* */
1829
1830 #define foreach_notrace_ip4_arp_error           \
1831 _(DROP)                                         \
1832 _(REQUEST_SENT)                                 \
1833 _(REPLICATE_DROP)                               \
1834 _(REPLICATE_FAIL)
1835
1836 clib_error_t *
1837 arp_notrace_init (vlib_main_t * vm)
1838 {
1839   vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, ip4_arp_node.index);
1840
1841   /* don't trace ARP request packets */
1842 #define _(a)                                    \
1843     vnet_pcap_drop_trace_filter_add_del         \
1844         (rt->errors[IP4_ARP_ERROR_##a],         \
1845          1 /* is_add */);
1846   foreach_notrace_ip4_arp_error;
1847 #undef _
1848   return 0;
1849 }
1850
1851 VLIB_INIT_FUNCTION (arp_notrace_init);
1852
1853
1854 /* Send an ARP request to see if given destination is reachable on given interface. */
1855 clib_error_t *
1856 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
1857 {
1858   vnet_main_t *vnm = vnet_get_main ();
1859   ip4_main_t *im = &ip4_main;
1860   ethernet_arp_header_t *h;
1861   ip4_address_t *src;
1862   ip_interface_address_t *ia;
1863   ip_adjacency_t *adj;
1864   vnet_hw_interface_t *hi;
1865   vnet_sw_interface_t *si;
1866   vlib_buffer_t *b;
1867   adj_index_t ai;
1868   u32 bi = 0;
1869
1870   si = vnet_get_sw_interface (vnm, sw_if_index);
1871
1872   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
1873     {
1874       return clib_error_return (0, "%U: interface %U down",
1875                                 format_ip4_address, dst,
1876                                 format_vnet_sw_if_index_name, vnm,
1877                                 sw_if_index);
1878     }
1879
1880   src =
1881     ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
1882   if (!src)
1883     {
1884       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
1885       return clib_error_return
1886         (0,
1887          "no matching interface address for destination %U (interface %U)",
1888          format_ip4_address, dst, format_vnet_sw_if_index_name, vnm,
1889          sw_if_index);
1890     }
1891
1892   h = vlib_packet_template_get_packet (vm,
1893                                        &im->ip4_arp_request_packet_template,
1894                                        &bi);
1895
1896   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1897   if (PREDICT_FALSE (!hi->hw_address))
1898     {
1899       return clib_error_return (0, "%U: interface %U do not support ip probe",
1900                                 format_ip4_address, dst,
1901                                 format_vnet_sw_if_index_name, vnm,
1902                                 sw_if_index);
1903     }
1904
1905   clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address,
1906                sizeof (h->ip4_over_ethernet[0].ethernet));
1907
1908   h->ip4_over_ethernet[0].ip4 = src[0];
1909   h->ip4_over_ethernet[1].ip4 = dst[0];
1910
1911   b = vlib_get_buffer (vm, bi);
1912   vnet_buffer (b)->sw_if_index[VLIB_RX] =
1913     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
1914
1915   ip46_address_t nh = {
1916     .ip4 = *dst,
1917   };
1918
1919   ai = adj_nbr_add_or_lock (FIB_PROTOCOL_IP4,
1920                             VNET_LINK_IP4, &nh, sw_if_index);
1921   adj = adj_get (ai);
1922
1923   /* Peer has been previously resolved, retrieve glean adj instead */
1924   if (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE)
1925     {
1926       adj_unlock (ai);
1927       ai = adj_glean_add_or_lock (FIB_PROTOCOL_IP4,
1928                                   VNET_LINK_IP4, sw_if_index, &nh);
1929       adj = adj_get (ai);
1930     }
1931
1932   /* Add encapsulation string for software interface (e.g. ethernet header). */
1933   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
1934   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
1935
1936   {
1937     vlib_frame_t *f = vlib_get_frame_to_node (vm, hi->output_node_index);
1938     u32 *to_next = vlib_frame_vector_args (f);
1939     to_next[0] = bi;
1940     f->n_vectors = 1;
1941     vlib_put_frame_to_node (vm, hi->output_node_index, f);
1942   }
1943
1944   adj_unlock (ai);
1945   return /* no error */ 0;
1946 }
1947
1948 typedef enum
1949 {
1950   IP4_REWRITE_NEXT_DROP,
1951   IP4_REWRITE_NEXT_ICMP_ERROR,
1952 } ip4_rewrite_next_t;
1953
1954 always_inline uword
1955 ip4_rewrite_inline (vlib_main_t * vm,
1956                     vlib_node_runtime_t * node,
1957                     vlib_frame_t * frame,
1958                     int do_counters, int is_midchain, int is_mcast)
1959 {
1960   ip_lookup_main_t *lm = &ip4_main.lookup_main;
1961   u32 *from = vlib_frame_vector_args (frame);
1962   u32 n_left_from, n_left_to_next, *to_next, next_index;
1963   vlib_node_runtime_t *error_node =
1964     vlib_node_get_runtime (vm, ip4_input_node.index);
1965
1966   n_left_from = frame->n_vectors;
1967   next_index = node->cached_next_index;
1968   u32 thread_index = vlib_get_thread_index ();
1969
1970   while (n_left_from > 0)
1971     {
1972       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1973
1974       while (n_left_from >= 4 && n_left_to_next >= 2)
1975         {
1976           ip_adjacency_t *adj0, *adj1;
1977           vlib_buffer_t *p0, *p1;
1978           ip4_header_t *ip0, *ip1;
1979           u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
1980           u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
1981           u32 tx_sw_if_index0, tx_sw_if_index1;
1982
1983           /* Prefetch next iteration. */
1984           {
1985             vlib_buffer_t *p2, *p3;
1986
1987             p2 = vlib_get_buffer (vm, from[2]);
1988             p3 = vlib_get_buffer (vm, from[3]);
1989
1990             vlib_prefetch_buffer_header (p2, STORE);
1991             vlib_prefetch_buffer_header (p3, STORE);
1992
1993             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1994             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1995           }
1996
1997           pi0 = to_next[0] = from[0];
1998           pi1 = to_next[1] = from[1];
1999
2000           from += 2;
2001           n_left_from -= 2;
2002           to_next += 2;
2003           n_left_to_next -= 2;
2004
2005           p0 = vlib_get_buffer (vm, pi0);
2006           p1 = vlib_get_buffer (vm, pi1);
2007
2008           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2009           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
2010
2011           /*
2012            * pre-fetch the per-adjacency counters
2013            */
2014           if (do_counters)
2015             {
2016               vlib_prefetch_combined_counter (&adjacency_counters,
2017                                               thread_index, adj_index0);
2018               vlib_prefetch_combined_counter (&adjacency_counters,
2019                                               thread_index, adj_index1);
2020             }
2021
2022           ip0 = vlib_buffer_get_current (p0);
2023           ip1 = vlib_buffer_get_current (p1);
2024
2025           error0 = error1 = IP4_ERROR_NONE;
2026           next0 = next1 = IP4_REWRITE_NEXT_DROP;
2027
2028           /* Decrement TTL & update checksum.
2029              Works either endian, so no need for byte swap. */
2030           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2031             {
2032               i32 ttl0 = ip0->ttl;
2033
2034               /* Input node should have reject packets with ttl 0. */
2035               ASSERT (ip0->ttl > 0);
2036
2037               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2038               checksum0 += checksum0 >= 0xffff;
2039
2040               ip0->checksum = checksum0;
2041               ttl0 -= 1;
2042               ip0->ttl = ttl0;
2043
2044               /*
2045                * If the ttl drops below 1 when forwarding, generate
2046                * an ICMP response.
2047                */
2048               if (PREDICT_FALSE (ttl0 <= 0))
2049                 {
2050                   error0 = IP4_ERROR_TIME_EXPIRED;
2051                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2052                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2053                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2054                                                0);
2055                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2056                 }
2057
2058               /* Verify checksum. */
2059               ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
2060                       (p0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2061             }
2062           else
2063             {
2064               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2065             }
2066           if (PREDICT_TRUE (!(p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2067             {
2068               i32 ttl1 = ip1->ttl;
2069
2070               /* Input node should have reject packets with ttl 0. */
2071               ASSERT (ip1->ttl > 0);
2072
2073               checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2074               checksum1 += checksum1 >= 0xffff;
2075
2076               ip1->checksum = checksum1;
2077               ttl1 -= 1;
2078               ip1->ttl = ttl1;
2079
2080               /*
2081                * If the ttl drops below 1 when forwarding, generate
2082                * an ICMP response.
2083                */
2084               if (PREDICT_FALSE (ttl1 <= 0))
2085                 {
2086                   error1 = IP4_ERROR_TIME_EXPIRED;
2087                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2088                   icmp4_error_set_vnet_buffer (p1, ICMP4_time_exceeded,
2089                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2090                                                0);
2091                   next1 = IP4_REWRITE_NEXT_ICMP_ERROR;
2092                 }
2093
2094               /* Verify checksum. */
2095               ASSERT ((ip1->checksum == ip4_header_checksum (ip1)) ||
2096                       (p1->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2097             }
2098           else
2099             {
2100               p1->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2101             }
2102
2103           /* Rewrite packet header and updates lengths. */
2104           adj0 = adj_get (adj_index0);
2105           adj1 = adj_get (adj_index1);
2106
2107           /* Worth pipelining. No guarantee that adj0,1 are hot... */
2108           rw_len0 = adj0[0].rewrite_header.data_bytes;
2109           rw_len1 = adj1[0].rewrite_header.data_bytes;
2110           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2111           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
2112
2113           /* Check MTU of outgoing interface. */
2114           if (vlib_buffer_length_in_chain (vm, p0) >
2115               adj0[0].rewrite_header.max_l3_packet_bytes)
2116             {
2117               error0 = IP4_ERROR_MTU_EXCEEDED;
2118               next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2119               icmp4_error_set_vnet_buffer
2120                 (p0, ICMP4_destination_unreachable,
2121                  ICMP4_destination_unreachable_fragmentation_needed_and_dont_fragment_set,
2122                  0);
2123             }
2124           if (vlib_buffer_length_in_chain (vm, p1) >
2125               adj1[0].rewrite_header.max_l3_packet_bytes)
2126             {
2127               error1 = IP4_ERROR_MTU_EXCEEDED;
2128               next1 = IP4_REWRITE_NEXT_ICMP_ERROR;
2129               icmp4_error_set_vnet_buffer
2130                 (p1, ICMP4_destination_unreachable,
2131                  ICMP4_destination_unreachable_fragmentation_needed_and_dont_fragment_set,
2132                  0);
2133             }
2134
2135           if (is_mcast)
2136             {
2137               error0 = ((adj0[0].rewrite_header.sw_if_index ==
2138                          vnet_buffer (p0)->sw_if_index[VLIB_RX]) ?
2139                         IP4_ERROR_SAME_INTERFACE : error0);
2140               error1 = ((adj1[0].rewrite_header.sw_if_index ==
2141                          vnet_buffer (p1)->sw_if_index[VLIB_RX]) ?
2142                         IP4_ERROR_SAME_INTERFACE : error1);
2143             }
2144
2145           p0->error = error_node->errors[error0];
2146           p1->error = error_node->errors[error1];
2147           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2148            * to see the IP headerr */
2149           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2150             {
2151               next0 = adj0[0].rewrite_header.next_index;
2152               p0->current_data -= rw_len0;
2153               p0->current_length += rw_len0;
2154               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2155               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2156
2157               if (PREDICT_FALSE
2158                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2159                 vnet_feature_arc_start (lm->output_feature_arc_index,
2160                                         tx_sw_if_index0, &next0, p0);
2161             }
2162           if (PREDICT_TRUE (error1 == IP4_ERROR_NONE))
2163             {
2164               next1 = adj1[0].rewrite_header.next_index;
2165               p1->current_data -= rw_len1;
2166               p1->current_length += rw_len1;
2167
2168               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
2169               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
2170
2171               if (PREDICT_FALSE
2172                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2173                 vnet_feature_arc_start (lm->output_feature_arc_index,
2174                                         tx_sw_if_index1, &next1, p1);
2175             }
2176
2177           /* Guess we are only writing on simple Ethernet header. */
2178           vnet_rewrite_two_headers (adj0[0], adj1[0],
2179                                     ip0, ip1, sizeof (ethernet_header_t));
2180
2181           /*
2182            * Bump the per-adjacency counters
2183            */
2184           if (do_counters)
2185             {
2186               vlib_increment_combined_counter
2187                 (&adjacency_counters,
2188                  thread_index,
2189                  adj_index0, 1,
2190                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2191
2192               vlib_increment_combined_counter
2193                 (&adjacency_counters,
2194                  thread_index,
2195                  adj_index1, 1,
2196                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
2197             }
2198
2199           if (is_midchain)
2200             {
2201               adj0->sub_type.midchain.fixup_func
2202                 (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
2203               adj1->sub_type.midchain.fixup_func
2204                 (vm, adj1, p1, adj0->sub_type.midchain.fixup_data);
2205             }
2206           if (is_mcast)
2207             {
2208               /*
2209                * copy bytes from the IP address into the MAC rewrite
2210                */
2211               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2212               vnet_fixup_one_header (adj1[0], &ip1->dst_address, ip1);
2213             }
2214
2215           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2216                                            to_next, n_left_to_next,
2217                                            pi0, pi1, next0, next1);
2218         }
2219
2220       while (n_left_from > 0 && n_left_to_next > 0)
2221         {
2222           ip_adjacency_t *adj0;
2223           vlib_buffer_t *p0;
2224           ip4_header_t *ip0;
2225           u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2226           u32 tx_sw_if_index0;
2227
2228           pi0 = to_next[0] = from[0];
2229
2230           p0 = vlib_get_buffer (vm, pi0);
2231
2232           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2233
2234           adj0 = adj_get (adj_index0);
2235
2236           ip0 = vlib_buffer_get_current (p0);
2237
2238           error0 = IP4_ERROR_NONE;
2239           next0 = IP4_REWRITE_NEXT_DROP;        /* drop on error */
2240
2241           /* Decrement TTL & update checksum. */
2242           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2243             {
2244               i32 ttl0 = ip0->ttl;
2245
2246               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2247
2248               checksum0 += checksum0 >= 0xffff;
2249
2250               ip0->checksum = checksum0;
2251
2252               ASSERT (ip0->ttl > 0);
2253
2254               ttl0 -= 1;
2255
2256               ip0->ttl = ttl0;
2257
2258               ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
2259                       (p0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2260
2261               if (PREDICT_FALSE (ttl0 <= 0))
2262                 {
2263                   /*
2264                    * If the ttl drops below 1 when forwarding, generate
2265                    * an ICMP response.
2266                    */
2267                   error0 = IP4_ERROR_TIME_EXPIRED;
2268                   next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2269                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2270                   icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2271                                                ICMP4_time_exceeded_ttl_exceeded_in_transit,
2272                                                0);
2273                 }
2274             }
2275           else
2276             {
2277               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2278             }
2279
2280           if (do_counters)
2281             vlib_prefetch_combined_counter (&adjacency_counters,
2282                                             thread_index, adj_index0);
2283
2284           /* Guess we are only writing on simple Ethernet header. */
2285           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2286           if (is_mcast)
2287             {
2288               /*
2289                * copy bytes from the IP address into the MAC rewrite
2290                */
2291               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2292             }
2293
2294           /* Update packet buffer attributes/set output interface. */
2295           rw_len0 = adj0[0].rewrite_header.data_bytes;
2296           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2297
2298           if (do_counters)
2299             vlib_increment_combined_counter
2300               (&adjacency_counters,
2301                thread_index, adj_index0, 1,
2302                vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2303
2304           /* Check MTU of outgoing interface. */
2305           if (vlib_buffer_length_in_chain (vm, p0) >
2306               adj0[0].rewrite_header.max_l3_packet_bytes)
2307             {
2308               error0 = IP4_ERROR_MTU_EXCEEDED;
2309               next0 = IP4_REWRITE_NEXT_ICMP_ERROR;
2310               icmp4_error_set_vnet_buffer
2311                 (p0, ICMP4_destination_unreachable,
2312                  ICMP4_destination_unreachable_fragmentation_needed_and_dont_fragment_set,
2313                  0);
2314             }
2315           if (is_mcast)
2316             {
2317               error0 = ((adj0[0].rewrite_header.sw_if_index ==
2318                          vnet_buffer (p0)->sw_if_index[VLIB_RX]) ?
2319                         IP4_ERROR_SAME_INTERFACE : error0);
2320             }
2321           p0->error = error_node->errors[error0];
2322
2323           /* Don't adjust the buffer for ttl issue; icmp-error node wants
2324            * to see the IP headerr */
2325           if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2326             {
2327               p0->current_data -= rw_len0;
2328               p0->current_length += rw_len0;
2329               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2330
2331               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2332               next0 = adj0[0].rewrite_header.next_index;
2333
2334               if (is_midchain)
2335                 {
2336                   adj0->sub_type.midchain.fixup_func
2337                     (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
2338                 }
2339
2340               if (PREDICT_FALSE
2341                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2342                 vnet_feature_arc_start (lm->output_feature_arc_index,
2343                                         tx_sw_if_index0, &next0, p0);
2344
2345             }
2346
2347           from += 1;
2348           n_left_from -= 1;
2349           to_next += 1;
2350           n_left_to_next -= 1;
2351
2352           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2353                                            to_next, n_left_to_next,
2354                                            pi0, next0);
2355         }
2356
2357       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2358     }
2359
2360   /* Need to do trace after rewrites to pick up new packet data. */
2361   if (node->flags & VLIB_NODE_FLAG_TRACE)
2362     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2363
2364   return frame->n_vectors;
2365 }
2366
2367
2368 /** @brief IPv4 rewrite node.
2369     @node ip4-rewrite
2370
2371     This is the IPv4 transit-rewrite node: decrement TTL, fix the ipv4
2372     header checksum, fetch the ip adjacency, check the outbound mtu,
2373     apply the adjacency rewrite, and send pkts to the adjacency
2374     rewrite header's rewrite_next_index.
2375
2376     @param vm vlib_main_t corresponding to the current thread
2377     @param node vlib_node_runtime_t
2378     @param frame vlib_frame_t whose contents should be dispatched
2379
2380     @par Graph mechanics: buffer metadata, next index usage
2381
2382     @em Uses:
2383     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
2384         - the rewrite adjacency index
2385     - <code>adj->lookup_next_index</code>
2386         - Must be IP_LOOKUP_NEXT_REWRITE or IP_LOOKUP_NEXT_ARP, otherwise
2387           the packet will be dropped.
2388     - <code>adj->rewrite_header</code>
2389         - Rewrite string length, rewrite string, next_index
2390
2391     @em Sets:
2392     - <code>b->current_data, b->current_length</code>
2393         - Updated net of applying the rewrite string
2394
2395     <em>Next Indices:</em>
2396     - <code> adj->rewrite_header.next_index </code>
2397       or @c ip4-drop
2398 */
2399 static uword
2400 ip4_rewrite (vlib_main_t * vm,
2401              vlib_node_runtime_t * node, vlib_frame_t * frame)
2402 {
2403   if (adj_are_counters_enabled ())
2404     return ip4_rewrite_inline (vm, node, frame, 1, 0, 0);
2405   else
2406     return ip4_rewrite_inline (vm, node, frame, 0, 0, 0);
2407 }
2408
2409 static uword
2410 ip4_midchain (vlib_main_t * vm,
2411               vlib_node_runtime_t * node, vlib_frame_t * frame)
2412 {
2413   if (adj_are_counters_enabled ())
2414     return ip4_rewrite_inline (vm, node, frame, 1, 1, 0);
2415   else
2416     return ip4_rewrite_inline (vm, node, frame, 0, 1, 0);
2417 }
2418
2419 static uword
2420 ip4_rewrite_mcast (vlib_main_t * vm,
2421                    vlib_node_runtime_t * node, vlib_frame_t * frame)
2422 {
2423   if (adj_are_counters_enabled ())
2424     return ip4_rewrite_inline (vm, node, frame, 1, 0, 1);
2425   else
2426     return ip4_rewrite_inline (vm, node, frame, 0, 0, 1);
2427 }
2428
2429 static uword
2430 ip4_mcast_midchain (vlib_main_t * vm,
2431                     vlib_node_runtime_t * node, vlib_frame_t * frame)
2432 {
2433   if (adj_are_counters_enabled ())
2434     return ip4_rewrite_inline (vm, node, frame, 1, 1, 1);
2435   else
2436     return ip4_rewrite_inline (vm, node, frame, 0, 1, 1);
2437 }
2438
2439 /* *INDENT-OFF* */
2440 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2441   .function = ip4_rewrite,
2442   .name = "ip4-rewrite",
2443   .vector_size = sizeof (u32),
2444
2445   .format_trace = format_ip4_rewrite_trace,
2446
2447   .n_next_nodes = 2,
2448   .next_nodes = {
2449     [IP4_REWRITE_NEXT_DROP] = "ip4-drop",
2450     [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2451   },
2452 };
2453 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite)
2454
2455 VLIB_REGISTER_NODE (ip4_rewrite_mcast_node) = {
2456   .function = ip4_rewrite_mcast,
2457   .name = "ip4-rewrite-mcast",
2458   .vector_size = sizeof (u32),
2459
2460   .format_trace = format_ip4_rewrite_trace,
2461   .sibling_of = "ip4-rewrite",
2462 };
2463 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_mcast_node, ip4_rewrite_mcast)
2464
2465 VLIB_REGISTER_NODE (ip4_mcast_midchain_node, static) = {
2466   .function = ip4_mcast_midchain,
2467   .name = "ip4-mcast-midchain",
2468   .vector_size = sizeof (u32),
2469
2470   .format_trace = format_ip4_rewrite_trace,
2471   .sibling_of = "ip4-rewrite",
2472 };
2473 VLIB_NODE_FUNCTION_MULTIARCH (ip4_mcast_midchain_node, ip4_mcast_midchain)
2474
2475 VLIB_REGISTER_NODE (ip4_midchain_node) = {
2476   .function = ip4_midchain,
2477   .name = "ip4-midchain",
2478   .vector_size = sizeof (u32),
2479   .format_trace = format_ip4_forward_next_trace,
2480   .sibling_of =  "ip4-rewrite",
2481 };
2482 VLIB_NODE_FUNCTION_MULTIARCH (ip4_midchain_node, ip4_midchain);
2483 /* *INDENT-ON */
2484
2485 int
2486 ip4_lookup_validate (ip4_address_t * a, u32 fib_index0)
2487 {
2488   ip4_fib_mtrie_t *mtrie0;
2489   ip4_fib_mtrie_leaf_t leaf0;
2490   u32 lbi0;
2491
2492   mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
2493
2494   leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, a);
2495   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
2496   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
2497
2498   lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2499
2500   return lbi0 == ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0), a);
2501 }
2502
2503 static clib_error_t *
2504 test_lookup_command_fn (vlib_main_t * vm,
2505                         unformat_input_t * input, vlib_cli_command_t * cmd)
2506 {
2507   ip4_fib_t *fib;
2508   u32 table_id = 0;
2509   f64 count = 1;
2510   u32 n;
2511   int i;
2512   ip4_address_t ip4_base_address;
2513   u64 errors = 0;
2514
2515   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2516     {
2517       if (unformat (input, "table %d", &table_id))
2518         {
2519           /* Make sure the entry exists. */
2520           fib = ip4_fib_get (table_id);
2521           if ((fib) && (fib->index != table_id))
2522             return clib_error_return (0, "<fib-index> %d does not exist",
2523                                       table_id);
2524         }
2525       else if (unformat (input, "count %f", &count))
2526         ;
2527
2528       else if (unformat (input, "%U",
2529                          unformat_ip4_address, &ip4_base_address))
2530         ;
2531       else
2532         return clib_error_return (0, "unknown input `%U'",
2533                                   format_unformat_error, input);
2534     }
2535
2536   n = count;
2537
2538   for (i = 0; i < n; i++)
2539     {
2540       if (!ip4_lookup_validate (&ip4_base_address, table_id))
2541         errors++;
2542
2543       ip4_base_address.as_u32 =
2544         clib_host_to_net_u32 (1 +
2545                               clib_net_to_host_u32 (ip4_base_address.as_u32));
2546     }
2547
2548   if (errors)
2549     vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
2550   else
2551     vlib_cli_output (vm, "No errors in %d lookups\n", n);
2552
2553   return 0;
2554 }
2555
2556 /*?
2557  * Perform a lookup of an IPv4 Address (or range of addresses) in the
2558  * given FIB table to determine if there is a conflict with the
2559  * adjacency table. The fib-id can be determined by using the
2560  * '<em>show ip fib</em>' command. If fib-id is not entered, default value
2561  * of 0 is used.
2562  *
2563  * @todo This command uses fib-id, other commands use table-id (not
2564  * just a name, they are different indexes). Would like to change this
2565  * to table-id for consistency.
2566  *
2567  * @cliexpar
2568  * Example of how to run the test lookup command:
2569  * @cliexstart{test lookup 172.16.1.1 table 1 count 2}
2570  * No errors in 2 lookups
2571  * @cliexend
2572 ?*/
2573 /* *INDENT-OFF* */
2574 VLIB_CLI_COMMAND (lookup_test_command, static) =
2575 {
2576   .path = "test lookup",
2577   .short_help = "test lookup <ipv4-addr> [table <fib-id>] [count <nn>]",
2578   .function = test_lookup_command_fn,
2579 };
2580 /* *INDENT-ON* */
2581
2582 int
2583 vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
2584 {
2585   u32 fib_index;
2586
2587   fib_index = fib_table_find (FIB_PROTOCOL_IP4, table_id);
2588
2589   if (~0 == fib_index)
2590     return VNET_API_ERROR_NO_SUCH_FIB;
2591
2592   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP4,
2593                                   flow_hash_config);
2594
2595   return 0;
2596 }
2597
2598 static clib_error_t *
2599 set_ip_flow_hash_command_fn (vlib_main_t * vm,
2600                              unformat_input_t * input,
2601                              vlib_cli_command_t * cmd)
2602 {
2603   int matched = 0;
2604   u32 table_id = 0;
2605   u32 flow_hash_config = 0;
2606   int rv;
2607
2608   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2609     {
2610       if (unformat (input, "table %d", &table_id))
2611         matched = 1;
2612 #define _(a,v) \
2613     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2614       foreach_flow_hash_bit
2615 #undef _
2616         else
2617         break;
2618     }
2619
2620   if (matched == 0)
2621     return clib_error_return (0, "unknown input `%U'",
2622                               format_unformat_error, input);
2623
2624   rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
2625   switch (rv)
2626     {
2627     case 0:
2628       break;
2629
2630     case VNET_API_ERROR_NO_SUCH_FIB:
2631       return clib_error_return (0, "no such FIB table %d", table_id);
2632
2633     default:
2634       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2635       break;
2636     }
2637
2638   return 0;
2639 }
2640
2641 /*?
2642  * Configure the set of IPv4 fields used by the flow hash.
2643  *
2644  * @cliexpar
2645  * Example of how to set the flow hash on a given table:
2646  * @cliexcmd{set ip flow-hash table 7 dst sport dport proto}
2647  * Example of display the configured flow hash:
2648  * @cliexstart{show ip fib}
2649  * ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2650  * 0.0.0.0/0
2651  *   unicast-ip4-chain
2652  *   [@0]: dpo-load-balance: [index:0 buckets:1 uRPF:0 to:[0:0]]
2653  *     [0] [@0]: dpo-drop ip6
2654  * 0.0.0.0/32
2655  *   unicast-ip4-chain
2656  *   [@0]: dpo-load-balance: [index:1 buckets:1 uRPF:1 to:[0:0]]
2657  *     [0] [@0]: dpo-drop ip6
2658  * 224.0.0.0/8
2659  *   unicast-ip4-chain
2660  *   [@0]: dpo-load-balance: [index:3 buckets:1 uRPF:3 to:[0:0]]
2661  *     [0] [@0]: dpo-drop ip6
2662  * 6.0.1.2/32
2663  *   unicast-ip4-chain
2664  *   [@0]: dpo-load-balance: [index:30 buckets:1 uRPF:29 to:[0:0]]
2665  *     [0] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
2666  * 7.0.0.1/32
2667  *   unicast-ip4-chain
2668  *   [@0]: dpo-load-balance: [index:31 buckets:4 uRPF:30 to:[0:0]]
2669  *     [0] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
2670  *     [1] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
2671  *     [2] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
2672  *     [3] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
2673  * 240.0.0.0/8
2674  *   unicast-ip4-chain
2675  *   [@0]: dpo-load-balance: [index:2 buckets:1 uRPF:2 to:[0:0]]
2676  *     [0] [@0]: dpo-drop ip6
2677  * 255.255.255.255/32
2678  *   unicast-ip4-chain
2679  *   [@0]: dpo-load-balance: [index:4 buckets:1 uRPF:4 to:[0:0]]
2680  *     [0] [@0]: dpo-drop ip6
2681  * ipv4-VRF:7, fib_index 1, flow hash: dst sport dport proto
2682  * 0.0.0.0/0
2683  *   unicast-ip4-chain
2684  *   [@0]: dpo-load-balance: [index:12 buckets:1 uRPF:11 to:[0:0]]
2685  *     [0] [@0]: dpo-drop ip6
2686  * 0.0.0.0/32
2687  *   unicast-ip4-chain
2688  *   [@0]: dpo-load-balance: [index:13 buckets:1 uRPF:12 to:[0:0]]
2689  *     [0] [@0]: dpo-drop ip6
2690  * 172.16.1.0/24
2691  *   unicast-ip4-chain
2692  *   [@0]: dpo-load-balance: [index:17 buckets:1 uRPF:16 to:[0:0]]
2693  *     [0] [@4]: ipv4-glean: af_packet0
2694  * 172.16.1.1/32
2695  *   unicast-ip4-chain
2696  *   [@0]: dpo-load-balance: [index:18 buckets:1 uRPF:17 to:[1:84]]
2697  *     [0] [@2]: dpo-receive: 172.16.1.1 on af_packet0
2698  * 172.16.1.2/32
2699  *   unicast-ip4-chain
2700  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
2701  *     [0] [@5]: ipv4 via 172.16.1.2 af_packet0: IP4: 02:fe:9e:70:7a:2b -> 26:a5:f6:9c:3a:36
2702  * 172.16.2.0/24
2703  *   unicast-ip4-chain
2704  *   [@0]: dpo-load-balance: [index:19 buckets:1 uRPF:18 to:[0:0]]
2705  *     [0] [@4]: ipv4-glean: af_packet1
2706  * 172.16.2.1/32
2707  *   unicast-ip4-chain
2708  *   [@0]: dpo-load-balance: [index:20 buckets:1 uRPF:19 to:[0:0]]
2709  *     [0] [@2]: dpo-receive: 172.16.2.1 on af_packet1
2710  * 224.0.0.0/8
2711  *   unicast-ip4-chain
2712  *   [@0]: dpo-load-balance: [index:15 buckets:1 uRPF:14 to:[0:0]]
2713  *     [0] [@0]: dpo-drop ip6
2714  * 240.0.0.0/8
2715  *   unicast-ip4-chain
2716  *   [@0]: dpo-load-balance: [index:14 buckets:1 uRPF:13 to:[0:0]]
2717  *     [0] [@0]: dpo-drop ip6
2718  * 255.255.255.255/32
2719  *   unicast-ip4-chain
2720  *   [@0]: dpo-load-balance: [index:16 buckets:1 uRPF:15 to:[0:0]]
2721  *     [0] [@0]: dpo-drop ip6
2722  * @cliexend
2723 ?*/
2724 /* *INDENT-OFF* */
2725 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) =
2726 {
2727   .path = "set ip flow-hash",
2728   .short_help =
2729   "set ip flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
2730   .function = set_ip_flow_hash_command_fn,
2731 };
2732 /* *INDENT-ON* */
2733
2734 int
2735 vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
2736                              u32 table_index)
2737 {
2738   vnet_main_t *vnm = vnet_get_main ();
2739   vnet_interface_main_t *im = &vnm->interface_main;
2740   ip4_main_t *ipm = &ip4_main;
2741   ip_lookup_main_t *lm = &ipm->lookup_main;
2742   vnet_classify_main_t *cm = &vnet_classify_main;
2743   ip4_address_t *if_addr;
2744
2745   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
2746     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
2747
2748   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
2749     return VNET_API_ERROR_NO_SUCH_ENTRY;
2750
2751   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
2752   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
2753
2754   if_addr = ip4_interface_first_address (ipm, sw_if_index, NULL);
2755
2756   if (NULL != if_addr)
2757     {
2758       fib_prefix_t pfx = {
2759         .fp_len = 32,
2760         .fp_proto = FIB_PROTOCOL_IP4,
2761         .fp_addr.ip4 = *if_addr,
2762       };
2763       u32 fib_index;
2764
2765       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2766                                                        sw_if_index);
2767
2768
2769       if (table_index != (u32) ~ 0)
2770         {
2771           dpo_id_t dpo = DPO_INVALID;
2772
2773           dpo_set (&dpo,
2774                    DPO_CLASSIFY,
2775                    DPO_PROTO_IP4,
2776                    classify_dpo_create (DPO_PROTO_IP4, table_index));
2777
2778           fib_table_entry_special_dpo_add (fib_index,
2779                                            &pfx,
2780                                            FIB_SOURCE_CLASSIFY,
2781                                            FIB_ENTRY_FLAG_NONE, &dpo);
2782           dpo_reset (&dpo);
2783         }
2784       else
2785         {
2786           fib_table_entry_special_remove (fib_index,
2787                                           &pfx, FIB_SOURCE_CLASSIFY);
2788         }
2789     }
2790
2791   return 0;
2792 }
2793
2794 static clib_error_t *
2795 set_ip_classify_command_fn (vlib_main_t * vm,
2796                             unformat_input_t * input,
2797                             vlib_cli_command_t * cmd)
2798 {
2799   u32 table_index = ~0;
2800   int table_index_set = 0;
2801   u32 sw_if_index = ~0;
2802   int rv;
2803
2804   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2805     {
2806       if (unformat (input, "table-index %d", &table_index))
2807         table_index_set = 1;
2808       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
2809                          vnet_get_main (), &sw_if_index))
2810         ;
2811       else
2812         break;
2813     }
2814
2815   if (table_index_set == 0)
2816     return clib_error_return (0, "classify table-index must be specified");
2817
2818   if (sw_if_index == ~0)
2819     return clib_error_return (0, "interface / subif must be specified");
2820
2821   rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
2822
2823   switch (rv)
2824     {
2825     case 0:
2826       break;
2827
2828     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
2829       return clib_error_return (0, "No such interface");
2830
2831     case VNET_API_ERROR_NO_SUCH_ENTRY:
2832       return clib_error_return (0, "No such classifier table");
2833     }
2834   return 0;
2835 }
2836
2837 /*?
2838  * Assign a classification table to an interface. The classification
2839  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
2840  * commands. Once the table is create, use this command to filter packets
2841  * on an interface.
2842  *
2843  * @cliexpar
2844  * Example of how to assign a classification table to an interface:
2845  * @cliexcmd{set ip classify intfc GigabitEthernet2/0/0 table-index 1}
2846 ?*/
2847 /* *INDENT-OFF* */
2848 VLIB_CLI_COMMAND (set_ip_classify_command, static) =
2849 {
2850     .path = "set ip classify",
2851     .short_help =
2852     "set ip classify intfc <interface> table-index <classify-idx>",
2853     .function = set_ip_classify_command_fn,
2854 };
2855 /* *INDENT-ON* */
2856
2857 static clib_error_t *
2858 ip4_config (vlib_main_t * vm, unformat_input_t * input)
2859 {
2860   ip4_main_t *im = &ip4_main;
2861   uword heapsize = 0;
2862
2863   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2864     {
2865       if (unformat (input, "heap-size %U", unformat_memory_size, &heapsize))
2866         ;
2867       else
2868         return clib_error_return (0,
2869                                   "invalid heap-size parameter `%U'",
2870                                   format_unformat_error, input);
2871     }
2872
2873   im->mtrie_heap_size = heapsize;
2874
2875   return 0;
2876 }
2877
2878 VLIB_EARLY_CONFIG_FUNCTION (ip4_config, "ip");
2879
2880 /*
2881  * fd.io coding-style-patch-verification: ON
2882  *
2883  * Local Variables:
2884  * eval: (c-set-style "gnu")
2885  * End:
2886  */