src/vnet/ip/ip4_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip4_forward.c: IP v4 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ip/ip_frag.h>
  43 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  44 #include <vnet/ethernet/arp_packet.h>   /* for ethernet_arp_header_t */
  45 #include <vnet/ppp/ppp.h>
  46 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  47 #include <vnet/api_errno.h>     /* for API error numbers */
  48 #include <vnet/fib/fib_table.h> /* for FIB table and entry creation */
  49 #include <vnet/fib/fib_entry.h> /* for FIB table and entry creation */
  50 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  51 #include <vnet/fib/ip4_fib.h>
  52 #include <vnet/dpo/load_balance.h>
  53 #include <vnet/dpo/load_balance_map.h>
  54 #include <vnet/dpo/classify_dpo.h>
  55 #include <vnet/mfib/mfib_table.h>       /* for mFIB table and entry creation */
  56
  57 #include <vnet/ip/ip4_forward.h>
  58
  59 /** @brief IPv4 lookup node.
  60     @node ip4-lookup
  61
  62     This is the main IPv4 lookup dispatch node.
  63
  64     @param vm vlib_main_t corresponding to the current thread
  65     @param node vlib_node_runtime_t
  66     @param frame vlib_frame_t whose contents should be dispatched
  67
  68     @par Graph mechanics: buffer metadata, next index usage
  69
  70     @em Uses:
  71     - <code>vnet_buffer(b)->sw_if_index[VLIB_RX]</code>
  72         - Indicates the @c sw_if_index value of the interface that the
  73           packet was received on.
  74     - <code>vnet_buffer(b)->sw_if_index[VLIB_TX]</code>
  75         - When the value is @c ~0 then the node performs a longest prefix
  76           match (LPM) for the packet destination address in the FIB attached
  77           to the receive interface.
  78         - Otherwise perform LPM for the packet destination address in the
  79           indicated FIB. In this case <code>[VLIB_TX]</code> is a FIB index
  80           value (0, 1, ...) and not a VRF id.
  81
  82     @em Sets:
  83     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
  84         - The lookup result adjacency index.
  85
  86     <em>Next Index:</em>
  87     - Dispatches the packet to the node index found in
  88       ip_adjacency_t @c adj->lookup_next_index
  89       (where @c adj is the lookup result adjacency).
  90 */
  91 VLIB_NODE_FN (ip4_lookup_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
  92                                 vlib_frame_t * frame)
  93 {
  94   return ip4_lookup_inline (vm, node, frame,
  95                             /* lookup_for_responses_to_locally_received_packets */
  96                             0);
  97
  98 }
  99
 100 static u8 *format_ip4_lookup_trace (u8 * s, va_list * args);
 101
 102 /* *INDENT-OFF* */
 103 VLIB_REGISTER_NODE (ip4_lookup_node) =
 104 {
 105   .name = "ip4-lookup",
 106   .vector_size = sizeof (u32),
 107   .format_trace = format_ip4_lookup_trace,
 108   .n_next_nodes = IP_LOOKUP_N_NEXT,
 109   .next_nodes = IP4_LOOKUP_NEXT_NODES,
 110 };
 111 /* *INDENT-ON* */
 112
 113 VLIB_NODE_FN (ip4_load_balance_node) (vlib_main_t * vm,
 114                                       vlib_node_runtime_t * node,
 115                                       vlib_frame_t * frame)
 116 {
 117   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 118   u32 n_left_from, n_left_to_next, *from, *to_next;
 119   ip_lookup_next_t next;
 120   u32 thread_index = vm->thread_index;
 121
 122   from = vlib_frame_vector_args (frame);
 123   n_left_from = frame->n_vectors;
 124   next = node->cached_next_index;
 125
 126   while (n_left_from > 0)
 127     {
 128       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 129
 130
 131       while (n_left_from >= 4 && n_left_to_next >= 2)
 132         {
 133           ip_lookup_next_t next0, next1;
 134           const load_balance_t *lb0, *lb1;
 135           vlib_buffer_t *p0, *p1;
 136           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 137           const ip4_header_t *ip0, *ip1;
 138           const dpo_id_t *dpo0, *dpo1;
 139
 140           /* Prefetch next iteration. */
 141           {
 142             vlib_buffer_t *p2, *p3;
 143
 144             p2 = vlib_get_buffer (vm, from[2]);
 145             p3 = vlib_get_buffer (vm, from[3]);
 146
 147             vlib_prefetch_buffer_header (p2, STORE);
 148             vlib_prefetch_buffer_header (p3, STORE);
 149
 150             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 151             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 152           }
 153
 154           pi0 = to_next[0] = from[0];
 155           pi1 = to_next[1] = from[1];
 156
 157           from += 2;
 158           n_left_from -= 2;
 159           to_next += 2;
 160           n_left_to_next -= 2;
 161
 162           p0 = vlib_get_buffer (vm, pi0);
 163           p1 = vlib_get_buffer (vm, pi1);
 164
 165           ip0 = vlib_buffer_get_current (p0);
 166           ip1 = vlib_buffer_get_current (p1);
 167           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 168           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 169
 170           lb0 = load_balance_get (lbi0);
 171           lb1 = load_balance_get (lbi1);
 172
 173           /*
 174            * this node is for via FIBs we can re-use the hash value from the
 175            * to node if present.
 176            * We don't want to use the same hash value at each level in the recursion
 177            * graph as that would lead to polarisation
 178            */
 179           hc0 = hc1 = 0;
 180
 181           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 182             {
 183               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 184                 {
 185                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 186                     vnet_buffer (p0)->ip.flow_hash >> 1;
 187                 }
 188               else
 189                 {
 190                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 191                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 192                 }
 193               dpo0 = load_balance_get_fwd_bucket
 194                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 195             }
 196           else
 197             {
 198               dpo0 = load_balance_get_bucket_i (lb0, 0);
 199             }
 200           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 201             {
 202               if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash))
 203                 {
 204                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 205                     vnet_buffer (p1)->ip.flow_hash >> 1;
 206                 }
 207               else
 208                 {
 209                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 210                     ip4_compute_flow_hash (ip1, lb1->lb_hash_config);
 211                 }
 212               dpo1 = load_balance_get_fwd_bucket
 213                 (lb1, (hc1 & (lb1->lb_n_buckets_minus_1)));
 214             }
 215           else
 216             {
 217               dpo1 = load_balance_get_bucket_i (lb1, 0);
 218             }
 219
 220           next0 = dpo0->dpoi_next_node;
 221           next1 = dpo1->dpoi_next_node;
 222
 223           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 224           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 225
 226           vlib_increment_combined_counter
 227             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 228           vlib_increment_combined_counter
 229             (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 230
 231           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 232                                            to_next, n_left_to_next,
 233                                            pi0, pi1, next0, next1);
 234         }
 235
 236       while (n_left_from > 0 && n_left_to_next > 0)
 237         {
 238           ip_lookup_next_t next0;
 239           const load_balance_t *lb0;
 240           vlib_buffer_t *p0;
 241           u32 pi0, lbi0, hc0;
 242           const ip4_header_t *ip0;
 243           const dpo_id_t *dpo0;
 244
 245           pi0 = from[0];
 246           to_next[0] = pi0;
 247           from += 1;
 248           to_next += 1;
 249           n_left_to_next -= 1;
 250           n_left_from -= 1;
 251
 252           p0 = vlib_get_buffer (vm, pi0);
 253
 254           ip0 = vlib_buffer_get_current (p0);
 255           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 256
 257           lb0 = load_balance_get (lbi0);
 258
 259           hc0 = 0;
 260           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 261             {
 262               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 263                 {
 264                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 265                     vnet_buffer (p0)->ip.flow_hash >> 1;
 266                 }
 267               else
 268                 {
 269                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 270                     ip4_compute_flow_hash (ip0, lb0->lb_hash_config);
 271                 }
 272               dpo0 = load_balance_get_fwd_bucket
 273                 (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 274             }
 275           else
 276             {
 277               dpo0 = load_balance_get_bucket_i (lb0, 0);
 278             }
 279
 280           next0 = dpo0->dpoi_next_node;
 281           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 282
 283           vlib_increment_combined_counter
 284             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 285
 286           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 287                                            to_next, n_left_to_next,
 288                                            pi0, next0);
 289         }
 290
 291       vlib_put_next_frame (vm, node, next, n_left_to_next);
 292     }
 293
 294   if (node->flags & VLIB_NODE_FLAG_TRACE)
 295     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
 296
 297   return frame->n_vectors;
 298 }
 299
 300 /* *INDENT-OFF* */
 301 VLIB_REGISTER_NODE (ip4_load_balance_node) =
 302 {
 303   .name = "ip4-load-balance",
 304   .vector_size = sizeof (u32),
 305   .sibling_of = "ip4-lookup",
 306   .format_trace = format_ip4_lookup_trace,
 307 };
 308 /* *INDENT-ON* */
 309
 310 #ifndef CLIB_MARCH_VARIANT
 311 /* get first interface address */
 312 ip4_address_t *
 313 ip4_interface_first_address (ip4_main_t * im, u32 sw_if_index,
 314                              ip_interface_address_t ** result_ia)
 315 {
 316   ip_lookup_main_t *lm = &im->lookup_main;
 317   ip_interface_address_t *ia = 0;
 318   ip4_address_t *result = 0;
 319
 320   /* *INDENT-OFF* */
 321   foreach_ip_interface_address
 322     (lm, ia, sw_if_index,
 323      1 /* honor unnumbered */ ,
 324      ({
 325        ip4_address_t * a =
 326          ip_interface_address_get_address (lm, ia);
 327        result = a;
 328        break;
 329      }));
 330   /* *INDENT-OFF* */
 331   if (result_ia)
 332     *result_ia = result ? ia : 0;
 333   return result;
 334 }
 335
 336 static void
 337 ip4_add_subnet_bcast_route (u32 fib_index,
 338                             fib_prefix_t *pfx,
 339                             u32 sw_if_index)
 340 {
 341   vnet_sw_interface_flags_t iflags;
 342
 343   iflags = vnet_sw_interface_get_flags(vnet_get_main(), sw_if_index);
 344
 345   fib_table_entry_special_remove(fib_index,
 346                                  pfx,
 347                                  FIB_SOURCE_INTERFACE);
 348
 349   if (iflags & VNET_SW_INTERFACE_FLAG_DIRECTED_BCAST)
 350     {
 351       fib_table_entry_update_one_path (fib_index, pfx,
 352                                        FIB_SOURCE_INTERFACE,
 353                                        FIB_ENTRY_FLAG_NONE,
 354                                        DPO_PROTO_IP4,
 355                                        /* No next-hop address */
 356                                        &ADJ_BCAST_ADDR,
 357                                        sw_if_index,
 358                                        // invalid FIB index
 359                                        ~0,
 360                                        1,
 361                                        // no out-label stack
 362                                        NULL,
 363                                        FIB_ROUTE_PATH_FLAG_NONE);
 364     }
 365   else
 366     {
 367         fib_table_entry_special_add(fib_index,
 368                                     pfx,
 369                                     FIB_SOURCE_INTERFACE,
 370                                     (FIB_ENTRY_FLAG_DROP |
 371                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 372     }
 373 }
 374
 375 static void
 376 ip4_add_interface_routes (u32 sw_if_index,
 377                           ip4_main_t * im, u32 fib_index,
 378                           ip_interface_address_t * a)
 379 {
 380   ip_lookup_main_t *lm = &im->lookup_main;
 381   ip4_address_t *address = ip_interface_address_get_address (lm, a);
 382   fib_prefix_t pfx = {
 383     .fp_len = a->address_length,
 384     .fp_proto = FIB_PROTOCOL_IP4,
 385     .fp_addr.ip4 = *address,
 386   };
 387
 388   if (pfx.fp_len <= 30)
 389     {
 390       /* a /30 or shorter - add a glean for the network address */
 391       fib_table_entry_update_one_path (fib_index, &pfx,
 392                                        FIB_SOURCE_INTERFACE,
 393                                        (FIB_ENTRY_FLAG_CONNECTED |
 394                                         FIB_ENTRY_FLAG_ATTACHED),
 395                                        DPO_PROTO_IP4,
 396                                        /* No next-hop address */
 397                                        NULL,
 398                                        sw_if_index,
 399                                        // invalid FIB index
 400                                        ~0,
 401                                        1,
 402                                        // no out-label stack
 403                                        NULL,
 404                                        FIB_ROUTE_PATH_FLAG_NONE);
 405
 406       /* Add the two broadcast addresses as drop */
 407       fib_prefix_t net_pfx = {
 408         .fp_len = 32,
 409         .fp_proto = FIB_PROTOCOL_IP4,
 410         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 411       };
 412       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 413         fib_table_entry_special_add(fib_index,
 414                                     &net_pfx,
 415                                     FIB_SOURCE_INTERFACE,
 416                                     (FIB_ENTRY_FLAG_DROP |
 417                                      FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT));
 418       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 419       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 420         ip4_add_subnet_bcast_route(fib_index, &net_pfx, sw_if_index);
 421     }
 422   else if (pfx.fp_len == 31)
 423     {
 424       u32 mask = clib_host_to_net_u32(1);
 425       fib_prefix_t net_pfx = pfx;
 426
 427       net_pfx.fp_len = 32;
 428       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 429
 430       /* a /31 - add the other end as an attached host */
 431       fib_table_entry_update_one_path (fib_index, &net_pfx,
 432                                        FIB_SOURCE_INTERFACE,
 433                                        (FIB_ENTRY_FLAG_ATTACHED),
 434                                        DPO_PROTO_IP4,
 435                                        &net_pfx.fp_addr,
 436                                        sw_if_index,
 437                                        // invalid FIB index
 438                                        ~0,
 439                                        1,
 440                                        NULL,
 441                                        FIB_ROUTE_PATH_FLAG_NONE);
 442     }
 443   pfx.fp_len = 32;
 444
 445   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 446     {
 447       u32 classify_table_index =
 448         lm->classify_table_index_by_sw_if_index[sw_if_index];
 449       if (classify_table_index != (u32) ~ 0)
 450         {
 451           dpo_id_t dpo = DPO_INVALID;
 452
 453           dpo_set (&dpo,
 454                    DPO_CLASSIFY,
 455                    DPO_PROTO_IP4,
 456                    classify_dpo_create (DPO_PROTO_IP4, classify_table_index));
 457
 458           fib_table_entry_special_dpo_add (fib_index,
 459                                            &pfx,
 460                                            FIB_SOURCE_CLASSIFY,
 461                                            FIB_ENTRY_FLAG_NONE, &dpo);
 462           dpo_reset (&dpo);
 463         }
 464     }
 465
 466   fib_table_entry_update_one_path (fib_index, &pfx,
 467                                    FIB_SOURCE_INTERFACE,
 468                                    (FIB_ENTRY_FLAG_CONNECTED |
 469                                     FIB_ENTRY_FLAG_LOCAL),
 470                                    DPO_PROTO_IP4,
 471                                    &pfx.fp_addr,
 472                                    sw_if_index,
 473                                    // invalid FIB index
 474                                    ~0,
 475                                    1, NULL,
 476                                    FIB_ROUTE_PATH_FLAG_NONE);
 477 }
 478
 479 static void
 480 ip4_del_interface_routes (ip4_main_t * im,
 481                           u32 fib_index,
 482                           ip4_address_t * address, u32 address_length)
 483 {
 484   fib_prefix_t pfx = {
 485     .fp_len = address_length,
 486     .fp_proto = FIB_PROTOCOL_IP4,
 487     .fp_addr.ip4 = *address,
 488   };
 489
 490   if (pfx.fp_len <= 30)
 491     {
 492       fib_prefix_t net_pfx = {
 493         .fp_len = 32,
 494         .fp_proto = FIB_PROTOCOL_IP4,
 495         .fp_addr.ip4.as_u32 = address->as_u32 & im->fib_masks[pfx.fp_len],
 496       };
 497       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 498         fib_table_entry_special_remove(fib_index,
 499                                        &net_pfx,
 500                                        FIB_SOURCE_INTERFACE);
 501       net_pfx.fp_addr.ip4.as_u32 |= ~im->fib_masks[pfx.fp_len];
 502       if (net_pfx.fp_addr.ip4.as_u32 != pfx.fp_addr.ip4.as_u32)
 503         fib_table_entry_special_remove(fib_index,
 504                                        &net_pfx,
 505                                        FIB_SOURCE_INTERFACE);
 506       fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 507     }
 508     else if (pfx.fp_len == 31)
 509     {
 510       u32 mask = clib_host_to_net_u32(1);
 511       fib_prefix_t net_pfx = pfx;
 512
 513       net_pfx.fp_len = 32;
 514       net_pfx.fp_addr.ip4.as_u32 ^= mask;
 515
 516       fib_table_entry_delete (fib_index, &net_pfx, FIB_SOURCE_INTERFACE);
 517     }
 518
 519   pfx.fp_len = 32;
 520   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 521 }
 522
 523 void
 524 ip4_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 525 {
 526   ip4_main_t *im = &ip4_main;
 527
 528   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 529
 530   /*
 531    * enable/disable only on the 1<->0 transition
 532    */
 533   if (is_enable)
 534     {
 535       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 536         return;
 537     }
 538   else
 539     {
 540       ASSERT (im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 541       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 542         return;
 543     }
 544   vnet_feature_enable_disable ("ip4-unicast", "ip4-not-enabled", sw_if_index,
 545                                !is_enable, 0, 0);
 546
 547
 548   vnet_feature_enable_disable ("ip4-multicast", "ip4-not-enabled",
 549                                sw_if_index, !is_enable, 0, 0);
 550 }
 551
 552 static clib_error_t *
 553 ip4_add_del_interface_address_internal (vlib_main_t * vm,
 554                                         u32 sw_if_index,
 555                                         ip4_address_t * address,
 556                                         u32 address_length, u32 is_del)
 557 {
 558   vnet_main_t *vnm = vnet_get_main ();
 559   ip4_main_t *im = &ip4_main;
 560   ip_lookup_main_t *lm = &im->lookup_main;
 561   clib_error_t *error = 0;
 562   u32 if_address_index, elts_before;
 563   ip4_address_fib_t ip4_af, *addr_fib = 0;
 564
 565   /* local0 interface doesn't support IP addressing  */
 566   if (sw_if_index == 0)
 567     {
 568       return
 569        clib_error_create ("local0 interface doesn't support IP addressing");
 570     }
 571
 572   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 573   ip4_addr_fib_init (&ip4_af, address,
 574                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 575   vec_add1 (addr_fib, ip4_af);
 576
 577   /*
 578    * there is no support for adj-fib handling in the presence of overlapping
 579    * subnets on interfaces. Easy fix - disallow overlapping subnets, like
 580    * most routers do.
 581    */
 582   /* *INDENT-OFF* */
 583   if (!is_del)
 584     {
 585       /* When adding an address check that it does not conflict
 586          with an existing address on any interface in this table. */
 587       ip_interface_address_t *ia;
 588       vnet_sw_interface_t *sif;
 589
 590       pool_foreach(sif, vnm->interface_main.sw_interfaces,
 591       ({
 592           if (im->fib_index_by_sw_if_index[sw_if_index] ==
 593               im->fib_index_by_sw_if_index[sif->sw_if_index])
 594             {
 595               foreach_ip_interface_address
 596                 (&im->lookup_main, ia, sif->sw_if_index,
 597                  0 /* honor unnumbered */ ,
 598                  ({
 599                    ip4_address_t * x =
 600                      ip_interface_address_get_address
 601                      (&im->lookup_main, ia);
 602                    if (ip4_destination_matches_route
 603                        (im, address, x, ia->address_length) ||
 604                        ip4_destination_matches_route (im,
 605                                                       x,
 606                                                       address,
 607                                                       address_length))
 608                      {
 609                        vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
 610
 611                        return
 612                          clib_error_create
 613                          ("failed to add %U which conflicts with %U for interface %U",
 614                           format_ip4_address_and_length, address,
 615                           address_length,
 616                           format_ip4_address_and_length, x,
 617                           ia->address_length,
 618                           format_vnet_sw_if_index_name, vnm,
 619                           sif->sw_if_index);
 620                      }
 621                  }));
 622             }
 623       }));
 624     }
 625   /* *INDENT-ON* */
 626
 627   elts_before = pool_elts (lm->if_address_pool);
 628
 629   error = ip_interface_address_add_del
 630     (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 631   if (error)
 632     goto done;
 633
 634   ip4_sw_interface_enable_disable (sw_if_index, !is_del);
 635
 636   if (is_del)
 637     ip4_del_interface_routes (im, ip4_af.fib_index, address, address_length);
 638   else
 639     ip4_add_interface_routes (sw_if_index,
 640                               im, ip4_af.fib_index,
 641                               pool_elt_at_index
 642                               (lm->if_address_pool, if_address_index));
 643
 644   /* If pool did not grow/shrink: add duplicate address. */
 645   if (elts_before != pool_elts (lm->if_address_pool))
 646     {
 647       ip4_add_del_interface_address_callback_t *cb;
 648       vec_foreach (cb, im->add_del_interface_address_callbacks)
 649         cb->function (im, cb->function_opaque, sw_if_index,
 650                       address, address_length, if_address_index, is_del);
 651     }
 652
 653 done:
 654   vec_free (addr_fib);
 655   return error;
 656 }
 657
 658 clib_error_t *
 659 ip4_add_del_interface_address (vlib_main_t * vm,
 660                                u32 sw_if_index,
 661                                ip4_address_t * address,
 662                                u32 address_length, u32 is_del)
 663 {
 664   return ip4_add_del_interface_address_internal
 665     (vm, sw_if_index, address, address_length, is_del);
 666 }
 667
 668 void
 669 ip4_directed_broadcast (u32 sw_if_index, u8 enable)
 670 {
 671   ip_interface_address_t *ia;
 672   ip4_main_t *im;
 673
 674   im = &ip4_main;
 675
 676   /*
 677    * when directed broadcast is enabled, the subnet braodcast route will forward
 678    * packets using an adjacency with a broadcast MAC. otherwise it drops
 679    */
 680   /* *INDENT-OFF* */
 681   foreach_ip_interface_address(&im->lookup_main, ia,
 682                                sw_if_index, 0,
 683      ({
 684        if (ia->address_length <= 30)
 685          {
 686            ip4_address_t *ipa;
 687
 688            ipa = ip_interface_address_get_address (&im->lookup_main, ia);
 689
 690            fib_prefix_t pfx = {
 691              .fp_len = 32,
 692              .fp_proto = FIB_PROTOCOL_IP4,
 693              .fp_addr = {
 694                .ip4.as_u32 = (ipa->as_u32 | ~im->fib_masks[ia->address_length]),
 695              },
 696            };
 697
 698            ip4_add_subnet_bcast_route
 699              (fib_table_get_index_for_sw_if_index(FIB_PROTOCOL_IP4,
 700                                                   sw_if_index),
 701               &pfx, sw_if_index);
 702          }
 703      }));
 704   /* *INDENT-ON* */
 705 }
 706 #endif
 707
 708 /* Built-in ip4 unicast rx feature path definition */
 709 /* *INDENT-OFF* */
 710 VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
 711 {
 712   .arc_name = "ip4-unicast",
 713   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 714   .last_in_arc = "ip4-lookup",
 715   .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
 716 };
 717
 718 VNET_FEATURE_INIT (ip4_flow_classify, static) =
 719 {
 720   .arc_name = "ip4-unicast",
 721   .node_name = "ip4-flow-classify",
 722   .runs_before = VNET_FEATURES ("ip4-inacl"),
 723 };
 724
 725 VNET_FEATURE_INIT (ip4_inacl, static) =
 726 {
 727   .arc_name = "ip4-unicast",
 728   .node_name = "ip4-inacl",
 729   .runs_before = VNET_FEATURES ("ip4-source-check-via-rx"),
 730 };
 731
 732 VNET_FEATURE_INIT (ip4_source_check_1, static) =
 733 {
 734   .arc_name = "ip4-unicast",
 735   .node_name = "ip4-source-check-via-rx",
 736   .runs_before = VNET_FEATURES ("ip4-source-check-via-any"),
 737 };
 738
 739 VNET_FEATURE_INIT (ip4_source_check_2, static) =
 740 {
 741   .arc_name = "ip4-unicast",
 742   .node_name = "ip4-source-check-via-any",
 743   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
 744 };
 745
 746 VNET_FEATURE_INIT (ip4_source_and_port_range_check_rx, static) =
 747 {
 748   .arc_name = "ip4-unicast",
 749   .node_name = "ip4-source-and-port-range-check-rx",
 750   .runs_before = VNET_FEATURES ("ip4-policer-classify"),
 751 };
 752
 753 VNET_FEATURE_INIT (ip4_policer_classify, static) =
 754 {
 755   .arc_name = "ip4-unicast",
 756   .node_name = "ip4-policer-classify",
 757   .runs_before = VNET_FEATURES ("ipsec4-input"),
 758 };
 759
 760 VNET_FEATURE_INIT (ip4_ipsec, static) =
 761 {
 762   .arc_name = "ip4-unicast",
 763   .node_name = "ipsec4-input",
 764   .runs_before = VNET_FEATURES ("vpath-input-ip4"),
 765 };
 766
 767 VNET_FEATURE_INIT (ip4_vpath, static) =
 768 {
 769   .arc_name = "ip4-unicast",
 770   .node_name = "vpath-input-ip4",
 771   .runs_before = VNET_FEATURES ("ip4-vxlan-bypass"),
 772 };
 773
 774 VNET_FEATURE_INIT (ip4_vxlan_bypass, static) =
 775 {
 776   .arc_name = "ip4-unicast",
 777   .node_name = "ip4-vxlan-bypass",
 778   .runs_before = VNET_FEATURES ("ip4-lookup"),
 779 };
 780
 781 VNET_FEATURE_INIT (ip4_not_enabled, static) =
 782 {
 783   .arc_name = "ip4-unicast",
 784   .node_name = "ip4-not-enabled",
 785   .runs_before = VNET_FEATURES ("ip4-lookup"),
 786 };
 787
 788 VNET_FEATURE_INIT (ip4_lookup, static) =
 789 {
 790   .arc_name = "ip4-unicast",
 791   .node_name = "ip4-lookup",
 792   .runs_before = 0,     /* not before any other features */
 793 };
 794
 795 /* Built-in ip4 multicast rx feature path definition */
 796 VNET_FEATURE_ARC_INIT (ip4_multicast, static) =
 797 {
 798   .arc_name = "ip4-multicast",
 799   .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
 800   .last_in_arc = "ip4-mfib-forward-lookup",
 801   .arc_index_ptr = &ip4_main.lookup_main.mcast_feature_arc_index,
 802 };
 803
 804 VNET_FEATURE_INIT (ip4_vpath_mc, static) =
 805 {
 806   .arc_name = "ip4-multicast",
 807   .node_name = "vpath-input-ip4",
 808   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
 809 };
 810
 811 VNET_FEATURE_INIT (ip4_mc_not_enabled, static) =
 812 {
 813   .arc_name = "ip4-multicast",
 814   .node_name = "ip4-not-enabled",
 815   .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
 816 };
 817
 818 VNET_FEATURE_INIT (ip4_lookup_mc, static) =
 819 {
 820   .arc_name = "ip4-multicast",
 821   .node_name = "ip4-mfib-forward-lookup",
 822   .runs_before = 0,     /* last feature */
 823 };
 824
 825 /* Source and port-range check ip4 tx feature path definition */
 826 VNET_FEATURE_ARC_INIT (ip4_output, static) =
 827 {
 828   .arc_name = "ip4-output",
 829   .start_nodes = VNET_FEATURES ("ip4-rewrite", "ip4-midchain", "ip4-dvr-dpo"),
 830   .last_in_arc = "interface-output",
 831   .arc_index_ptr = &ip4_main.lookup_main.output_feature_arc_index,
 832 };
 833
 834 VNET_FEATURE_INIT (ip4_source_and_port_range_check_tx, static) =
 835 {
 836   .arc_name = "ip4-output",
 837   .node_name = "ip4-source-and-port-range-check-tx",
 838   .runs_before = VNET_FEATURES ("ip4-outacl"),
 839 };
 840
 841 VNET_FEATURE_INIT (ip4_outacl, static) =
 842 {
 843   .arc_name = "ip4-output",
 844   .node_name = "ip4-outacl",
 845   .runs_before = VNET_FEATURES ("ipsec4-output"),
 846 };
 847
 848 VNET_FEATURE_INIT (ip4_ipsec_output, static) =
 849 {
 850   .arc_name = "ip4-output",
 851   .node_name = "ipsec4-output",
 852   .runs_before = VNET_FEATURES ("interface-output"),
 853 };
 854
 855 /* Built-in ip4 tx feature path definition */
 856 VNET_FEATURE_INIT (ip4_interface_output, static) =
 857 {
 858   .arc_name = "ip4-output",
 859   .node_name = "interface-output",
 860   .runs_before = 0,     /* not before any other features */
 861 };
 862 /* *INDENT-ON* */
 863
 864 static clib_error_t *
 865 ip4_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 866 {
 867   ip4_main_t *im = &ip4_main;
 868
 869   /* Fill in lookup tables with default table (0). */
 870   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 871   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 872
 873   if (!is_add)
 874     {
 875       ip4_main_t *im4 = &ip4_main;
 876       ip_lookup_main_t *lm4 = &im4->lookup_main;
 877       ip_interface_address_t *ia = 0;
 878       ip4_address_t *address;
 879       vlib_main_t *vm = vlib_get_main ();
 880
 881       vnet_sw_interface_update_unnumbered (sw_if_index, ~0, 0);
 882       /* *INDENT-OFF* */
 883       foreach_ip_interface_address (lm4, ia, sw_if_index, 0,
 884       ({
 885         address = ip_interface_address_get_address (lm4, ia);
 886         ip4_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
 887       }));
 888       /* *INDENT-ON* */
 889     }
 890
 891   vnet_feature_enable_disable ("ip4-unicast", "ip4-not-enabled", sw_if_index,
 892                                is_add, 0, 0);
 893
 894   vnet_feature_enable_disable ("ip4-multicast", "ip4-not-enabled",
 895                                sw_if_index, is_add, 0, 0);
 896
 897   return /* no error */ 0;
 898 }
 899
 900 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
 901
 902 /* Global IP4 main. */
 903 ip4_main_t ip4_main;
 904
 905 static clib_error_t *
 906 ip4_lookup_init (vlib_main_t * vm)
 907 {
 908   ip4_main_t *im = &ip4_main;
 909   clib_error_t *error;
 910   uword i;
 911
 912   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
 913     return error;
 914   if ((error = vlib_call_init_function (vm, ip4_mtrie_module_init)))
 915     return (error);
 916   if ((error = vlib_call_init_function (vm, fib_module_init)))
 917     return error;
 918   if ((error = vlib_call_init_function (vm, mfib_module_init)))
 919     return error;
 920
 921   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
 922     {
 923       u32 m;
 924
 925       if (i < 32)
 926         m = pow2_mask (i) << (32 - i);
 927       else
 928         m = ~0;
 929       im->fib_masks[i] = clib_host_to_net_u32 (m);
 930     }
 931
 932   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
 933
 934   /* Create FIB with index 0 and table id of 0. */
 935   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0,
 936                                      FIB_SOURCE_DEFAULT_ROUTE);
 937   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0,
 938                                       MFIB_SOURCE_DEFAULT_ROUTE);
 939
 940   {
 941     pg_node_t *pn;
 942     pn = pg_get_node (ip4_lookup_node.index);
 943     pn->unformat_edit = unformat_pg_ip4_header;
 944   }
 945
 946   {
 947     ethernet_arp_header_t h;
 948
 949     clib_memset (&h, 0, sizeof (h));
 950
 951     /* Set target ethernet address to all zeros. */
 952     clib_memset (h.ip4_over_ethernet[1].ethernet, 0,
 953                  sizeof (h.ip4_over_ethernet[1].ethernet));
 954
 955 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
 956 #define _8(f,v) h.f = v;
 957     _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
 958     _16 (l3_type, ETHERNET_TYPE_IP4);
 959     _8 (n_l2_address_bytes, 6);
 960     _8 (n_l3_address_bytes, 4);
 961     _16 (opcode, ETHERNET_ARP_OPCODE_request);
 962 #undef _16
 963 #undef _8
 964
 965     vlib_packet_template_init (vm, &im->ip4_arp_request_packet_template,
 966                                /* data */ &h,
 967                                sizeof (h),
 968                                /* alloc chunk size */ 8,
 969                                "ip4 arp");
 970   }
 971
 972   return error;
 973 }
 974
 975 VLIB_INIT_FUNCTION (ip4_lookup_init);
 976
 977 typedef struct
 978 {
 979   /* Adjacency taken. */
 980   u32 dpo_index;
 981   u32 flow_hash;
 982   u32 fib_index;
 983
 984   /* Packet data, possibly *after* rewrite. */
 985   u8 packet_data[64 - 1 * sizeof (u32)];
 986 }
 987 ip4_forward_next_trace_t;
 988
 989 #ifndef CLIB_MARCH_VARIANT
 990 u8 *
 991 format_ip4_forward_next_trace (u8 * s, va_list * args)
 992 {
 993   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 994   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 995   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
 996   u32 indent = format_get_indent (s);
 997   s = format (s, "%U%U",
 998               format_white_space, indent,
 999               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1000   return s;
1001 }
1002 #endif
1003
1004 static u8 *
1005 format_ip4_lookup_trace (u8 * s, va_list * args)
1006 {
1007   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1008   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1009   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1010   u32 indent = format_get_indent (s);
1011
1012   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
1013               t->fib_index, t->dpo_index, t->flow_hash);
1014   s = format (s, "\n%U%U",
1015               format_white_space, indent,
1016               format_ip4_header, t->packet_data, sizeof (t->packet_data));
1017   return s;
1018 }
1019
1020 static u8 *
1021 format_ip4_rewrite_trace (u8 * s, va_list * args)
1022 {
1023   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1024   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1025   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1026   u32 indent = format_get_indent (s);
1027
1028   s = format (s, "tx_sw_if_index %d dpo-idx %d : %U flow hash: 0x%08x",
1029               t->fib_index, t->dpo_index, format_ip_adjacency,
1030               t->dpo_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
1031   s = format (s, "\n%U%U",
1032               format_white_space, indent,
1033               format_ip_adjacency_packet_data,
1034               t->dpo_index, t->packet_data, sizeof (t->packet_data));
1035   return s;
1036 }
1037
1038 #ifndef CLIB_MARCH_VARIANT
1039 /* Common trace function for all ip4-forward next nodes. */
1040 void
1041 ip4_forward_next_trace (vlib_main_t * vm,
1042                         vlib_node_runtime_t * node,
1043                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
1044 {
1045   u32 *from, n_left;
1046   ip4_main_t *im = &ip4_main;
1047
1048   n_left = frame->n_vectors;
1049   from = vlib_frame_vector_args (frame);
1050
1051   while (n_left >= 4)
1052     {
1053       u32 bi0, bi1;
1054       vlib_buffer_t *b0, *b1;
1055       ip4_forward_next_trace_t *t0, *t1;
1056
1057       /* Prefetch next iteration. */
1058       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1059       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1060
1061       bi0 = from[0];
1062       bi1 = from[1];
1063
1064       b0 = vlib_get_buffer (vm, bi0);
1065       b1 = vlib_get_buffer (vm, bi1);
1066
1067       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1068         {
1069           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1070           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1071           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1072           t0->fib_index =
1073             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1074              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1075             vec_elt (im->fib_index_by_sw_if_index,
1076                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1077
1078           clib_memcpy_fast (t0->packet_data,
1079                             vlib_buffer_get_current (b0),
1080                             sizeof (t0->packet_data));
1081         }
1082       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1083         {
1084           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1085           t1->dpo_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1086           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1087           t1->fib_index =
1088             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
1089              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
1090             vec_elt (im->fib_index_by_sw_if_index,
1091                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
1092           clib_memcpy_fast (t1->packet_data, vlib_buffer_get_current (b1),
1093                             sizeof (t1->packet_data));
1094         }
1095       from += 2;
1096       n_left -= 2;
1097     }
1098
1099   while (n_left >= 1)
1100     {
1101       u32 bi0;
1102       vlib_buffer_t *b0;
1103       ip4_forward_next_trace_t *t0;
1104
1105       bi0 = from[0];
1106
1107       b0 = vlib_get_buffer (vm, bi0);
1108
1109       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1110         {
1111           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1112           t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1113           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1114           t0->fib_index =
1115             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1116              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1117             vec_elt (im->fib_index_by_sw_if_index,
1118                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1119           clib_memcpy_fast (t0->packet_data, vlib_buffer_get_current (b0),
1120                             sizeof (t0->packet_data));
1121         }
1122       from += 1;
1123       n_left -= 1;
1124     }
1125 }
1126
1127 /* Compute TCP/UDP/ICMP4 checksum in software. */
1128 u16
1129 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1130                               ip4_header_t * ip0)
1131 {
1132   ip_csum_t sum0;
1133   u32 ip_header_length, payload_length_host_byte_order;
1134   u32 n_this_buffer, n_bytes_left, n_ip_bytes_this_buffer;
1135   u16 sum16;
1136   void *data_this_buffer;
1137
1138   /* Initialize checksum with ip header. */
1139   ip_header_length = ip4_header_bytes (ip0);
1140   payload_length_host_byte_order =
1141     clib_net_to_host_u16 (ip0->length) - ip_header_length;
1142   sum0 =
1143     clib_host_to_net_u32 (payload_length_host_byte_order +
1144                           (ip0->protocol << 16));
1145
1146   if (BITS (uword) == 32)
1147     {
1148       sum0 =
1149         ip_csum_with_carry (sum0,
1150                             clib_mem_unaligned (&ip0->src_address, u32));
1151       sum0 =
1152         ip_csum_with_carry (sum0,
1153                             clib_mem_unaligned (&ip0->dst_address, u32));
1154     }
1155   else
1156     sum0 =
1157       ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1158
1159   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1160   data_this_buffer = (void *) ip0 + ip_header_length;
1161   n_ip_bytes_this_buffer =
1162     p0->current_length - (((u8 *) ip0 - p0->data) - p0->current_data);
1163   if (n_this_buffer + ip_header_length > n_ip_bytes_this_buffer)
1164     {
1165       n_this_buffer = n_ip_bytes_this_buffer > ip_header_length ?
1166         n_ip_bytes_this_buffer - ip_header_length : 0;
1167     }
1168   while (1)
1169     {
1170       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1171       n_bytes_left -= n_this_buffer;
1172       if (n_bytes_left == 0)
1173         break;
1174
1175       ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1176       p0 = vlib_get_buffer (vm, p0->next_buffer);
1177       data_this_buffer = vlib_buffer_get_current (p0);
1178       n_this_buffer = p0->current_length;
1179     }
1180
1181   sum16 = ~ip_csum_fold (sum0);
1182
1183   return sum16;
1184 }
1185
1186 u32
1187 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1188 {
1189   ip4_header_t *ip0 = vlib_buffer_get_current (p0);
1190   udp_header_t *udp0;
1191   u16 sum16;
1192
1193   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1194           || ip0->protocol == IP_PROTOCOL_UDP);
1195
1196   udp0 = (void *) (ip0 + 1);
1197   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1198     {
1199       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1200                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
1201       return p0->flags;
1202     }
1203
1204   sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1205
1206   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1207                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
1208
1209   return p0->flags;
1210 }
1211 #endif
1212
1213 /* *INDENT-OFF* */
1214 VNET_FEATURE_ARC_INIT (ip4_local) =
1215 {
1216   .arc_name  = "ip4-local",
1217   .start_nodes = VNET_FEATURES ("ip4-local"),
1218   .last_in_arc = "ip4-local-end-of-arc",
1219 };
1220 /* *INDENT-ON* */
1221
1222 static inline void
1223 ip4_local_l4_csum_validate (vlib_main_t * vm, vlib_buffer_t * p,
1224                             ip4_header_t * ip, u8 is_udp, u8 * error,
1225                             u8 * good_tcp_udp)
1226 {
1227   u32 flags0;
1228   flags0 = ip4_tcp_udp_validate_checksum (vm, p);
1229   *good_tcp_udp = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1230   if (is_udp)
1231     {
1232       udp_header_t *udp;
1233       u32 ip_len, udp_len;
1234       i32 len_diff;
1235       udp = ip4_next_header (ip);
1236       /* Verify UDP length. */
1237       ip_len = clib_net_to_host_u16 (ip->length);
1238       udp_len = clib_net_to_host_u16 (udp->length);
1239
1240       len_diff = ip_len - udp_len;
1241       *good_tcp_udp &= len_diff >= 0;
1242       *error = len_diff < 0 ? IP4_ERROR_UDP_LENGTH : *error;
1243     }
1244 }
1245
1246 #define ip4_local_csum_is_offloaded(_b)                                 \
1247     _b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM                         \
1248         || _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM
1249
1250 #define ip4_local_need_csum_check(is_tcp_udp, _b)                       \
1251     (is_tcp_udp && !(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED     \
1252         || ip4_local_csum_is_offloaded (_b)))
1253
1254 #define ip4_local_csum_is_valid(_b)                                     \
1255     (_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT                      \
1256         || (ip4_local_csum_is_offloaded (_b))) != 0
1257
1258 static inline void
1259 ip4_local_check_l4_csum (vlib_main_t * vm, vlib_buffer_t * b,
1260                          ip4_header_t * ih, u8 * error)
1261 {
1262   u8 is_udp, is_tcp_udp, good_tcp_udp;
1263
1264   is_udp = ih->protocol == IP_PROTOCOL_UDP;
1265   is_tcp_udp = is_udp || ih->protocol == IP_PROTOCOL_TCP;
1266
1267   if (PREDICT_FALSE (ip4_local_need_csum_check (is_tcp_udp, b)))
1268     ip4_local_l4_csum_validate (vm, b, ih, is_udp, error, &good_tcp_udp);
1269   else
1270     good_tcp_udp = ip4_local_csum_is_valid (b);
1271
1272   ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1273   *error = (is_tcp_udp && !good_tcp_udp
1274             ? IP4_ERROR_TCP_CHECKSUM + is_udp : *error);
1275 }
1276
1277 static inline void
1278 ip4_local_check_l4_csum_x2 (vlib_main_t * vm, vlib_buffer_t ** b,
1279                             ip4_header_t ** ih, u8 * error)
1280 {
1281   u8 is_udp[2], is_tcp_udp[2], good_tcp_udp[2];
1282
1283   is_udp[0] = ih[0]->protocol == IP_PROTOCOL_UDP;
1284   is_udp[1] = ih[1]->protocol == IP_PROTOCOL_UDP;
1285
1286   is_tcp_udp[0] = is_udp[0] || ih[0]->protocol == IP_PROTOCOL_TCP;
1287   is_tcp_udp[1] = is_udp[1] || ih[1]->protocol == IP_PROTOCOL_TCP;
1288
1289   good_tcp_udp[0] = ip4_local_csum_is_valid (b[0]);
1290   good_tcp_udp[1] = ip4_local_csum_is_valid (b[1]);
1291
1292   if (PREDICT_FALSE (ip4_local_need_csum_check (is_tcp_udp[0], b[0])
1293                      || ip4_local_need_csum_check (is_tcp_udp[1], b[1])))
1294     {
1295       if (is_tcp_udp[0])
1296         ip4_local_l4_csum_validate (vm, b[0], ih[0], is_udp[0], &error[0],
1297                                     &good_tcp_udp[0]);
1298       if (is_tcp_udp[1])
1299         ip4_local_l4_csum_validate (vm, b[1], ih[1], is_udp[1], &error[1],
1300                                     &good_tcp_udp[1]);
1301     }
1302
1303   error[0] = (is_tcp_udp[0] && !good_tcp_udp[0] ?
1304               IP4_ERROR_TCP_CHECKSUM + is_udp[0] : error[0]);
1305   error[1] = (is_tcp_udp[1] && !good_tcp_udp[1] ?
1306               IP4_ERROR_TCP_CHECKSUM + is_udp[1] : error[1]);
1307 }
1308
1309 static inline void
1310 ip4_local_set_next_and_error (vlib_node_runtime_t * error_node,
1311                               vlib_buffer_t * b, u16 * next, u8 error,
1312                               u8 head_of_feature_arc)
1313 {
1314   u8 arc_index = vnet_feat_arc_ip4_local.feature_arc_index;
1315   u32 next_index;
1316
1317   *next = error != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : *next;
1318   b->error = error ? error_node->errors[error] : 0;
1319   if (head_of_feature_arc)
1320     {
1321       next_index = *next;
1322       if (PREDICT_TRUE (error == (u8) IP4_ERROR_UNKNOWN_PROTOCOL))
1323         {
1324           vnet_feature_arc_start (arc_index,
1325                                   vnet_buffer (b)->sw_if_index[VLIB_RX],
1326                                   &next_index, b);
1327           *next = next_index;
1328         }
1329     }
1330 }
1331
1332 typedef struct
1333 {
1334   ip4_address_t src;
1335   u32 lbi;
1336   u8 error;
1337 } ip4_local_last_check_t;
1338
1339 static inline void
1340 ip4_local_check_src (vlib_buffer_t * b, ip4_header_t * ip0,
1341                      ip4_local_last_check_t * last_check, u8 * error0)
1342 {
1343   ip4_fib_mtrie_leaf_t leaf0;
1344   ip4_fib_mtrie_t *mtrie0;
1345   const dpo_id_t *dpo0;
1346   load_balance_t *lb0;
1347   u32 lbi0;
1348
1349   vnet_buffer (b)->ip.fib_index =
1350     vnet_buffer (b)->sw_if_index[VLIB_TX] != ~0 ?
1351     vnet_buffer (b)->sw_if_index[VLIB_TX] : vnet_buffer (b)->ip.fib_index;
1352
1353   if (PREDICT_FALSE (last_check->src.as_u32 != ip0->src_address.as_u32))
1354     {
1355       mtrie0 = &ip4_fib_get (vnet_buffer (b)->ip.fib_index)->mtrie;
1356       leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, &ip0->src_address);
1357       leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1358       leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1359       lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1360
1361       vnet_buffer (b)->ip.adj_index[VLIB_TX] = lbi0;
1362       vnet_buffer (b)->ip.adj_index[VLIB_RX] = lbi0;
1363
1364       lb0 = load_balance_get (lbi0);
1365       dpo0 = load_balance_get_bucket_i (lb0, 0);
1366
1367       /*
1368        * Must have a route to source otherwise we drop the packet.
1369        * ip4 broadcasts are accepted, e.g. to make dhcp client work
1370        *
1371        * The checks are:
1372        *  - the source is a recieve => it's from us => bogus, do this
1373        *    first since it sets a different error code.
1374        *  - uRPF check for any route to source - accept if passes.
1375        *  - allow packets destined to the broadcast address from unknown sources
1376        */
1377
1378       *error0 = ((*error0 == IP4_ERROR_UNKNOWN_PROTOCOL
1379                   && dpo0->dpoi_type == DPO_RECEIVE) ?
1380                  IP4_ERROR_SPOOFED_LOCAL_PACKETS : *error0);
1381       *error0 = ((*error0 == IP4_ERROR_UNKNOWN_PROTOCOL
1382                   && !fib_urpf_check_size (lb0->lb_urpf)
1383                   && ip0->dst_address.as_u32 != 0xFFFFFFFF) ?
1384                  IP4_ERROR_SRC_LOOKUP_MISS : *error0);
1385
1386       last_check->src.as_u32 = ip0->src_address.as_u32;
1387       last_check->lbi = lbi0;
1388       last_check->error = *error0;
1389     }
1390   else
1391     {
1392       vnet_buffer (b)->ip.adj_index[VLIB_TX] = last_check->lbi;
1393       vnet_buffer (b)->ip.adj_index[VLIB_RX] = last_check->lbi;
1394       *error0 = last_check->error;
1395     }
1396 }
1397
1398 static inline void
1399 ip4_local_check_src_x2 (vlib_buffer_t ** b, ip4_header_t ** ip,
1400                         ip4_local_last_check_t * last_check, u8 * error)
1401 {
1402   ip4_fib_mtrie_leaf_t leaf[2];
1403   ip4_fib_mtrie_t *mtrie[2];
1404   const dpo_id_t *dpo[2];
1405   load_balance_t *lb[2];
1406   u32 not_last_hit = 0;
1407   u32 lbi[2];
1408
1409   not_last_hit |= ip[0]->src_address.as_u32 ^ last_check->src.as_u32;
1410   not_last_hit |= ip[1]->src_address.as_u32 ^ last_check->src.as_u32;
1411
1412   vnet_buffer (b[0])->ip.fib_index =
1413     vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ?
1414     vnet_buffer (b[0])->sw_if_index[VLIB_TX] :
1415     vnet_buffer (b[0])->ip.fib_index;
1416
1417   vnet_buffer (b[1])->ip.fib_index =
1418     vnet_buffer (b[1])->sw_if_index[VLIB_TX] != ~0 ?
1419     vnet_buffer (b[1])->sw_if_index[VLIB_TX] :
1420     vnet_buffer (b[1])->ip.fib_index;
1421
1422   if (PREDICT_FALSE (not_last_hit))
1423     {
1424       mtrie[0] = &ip4_fib_get (vnet_buffer (b[0])->ip.fib_index)->mtrie;
1425       mtrie[1] = &ip4_fib_get (vnet_buffer (b[1])->ip.fib_index)->mtrie;
1426
1427       leaf[0] = ip4_fib_mtrie_lookup_step_one (mtrie[0], &ip[0]->src_address);
1428       leaf[1] = ip4_fib_mtrie_lookup_step_one (mtrie[1], &ip[1]->src_address);
1429
1430       leaf[0] = ip4_fib_mtrie_lookup_step (mtrie[0], leaf[0],
1431                                            &ip[0]->src_address, 2);
1432       leaf[1] = ip4_fib_mtrie_lookup_step (mtrie[1], leaf[1],
1433                                            &ip[1]->src_address, 2);
1434
1435       leaf[0] = ip4_fib_mtrie_lookup_step (mtrie[0], leaf[0],
1436                                            &ip[0]->src_address, 3);
1437       leaf[1] = ip4_fib_mtrie_lookup_step (mtrie[1], leaf[1],
1438                                            &ip[1]->src_address, 3);
1439
1440       lbi[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf[0]);
1441       lbi[1] = ip4_fib_mtrie_leaf_get_adj_index (leaf[1]);
1442
1443       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = lbi[0];
1444       vnet_buffer (b[0])->ip.adj_index[VLIB_RX] = lbi[0];
1445
1446       vnet_buffer (b[1])->ip.adj_index[VLIB_TX] = lbi[1];
1447       vnet_buffer (b[1])->ip.adj_index[VLIB_RX] = lbi[1];
1448
1449       lb[0] = load_balance_get (lbi[0]);
1450       lb[1] = load_balance_get (lbi[1]);
1451
1452       dpo[0] = load_balance_get_bucket_i (lb[0], 0);
1453       dpo[1] = load_balance_get_bucket_i (lb[1], 0);
1454
1455       error[0] = ((error[0] == IP4_ERROR_UNKNOWN_PROTOCOL &&
1456                    dpo[0]->dpoi_type == DPO_RECEIVE) ?
1457                   IP4_ERROR_SPOOFED_LOCAL_PACKETS : error[0]);
1458       error[0] = ((error[0] == IP4_ERROR_UNKNOWN_PROTOCOL &&
1459                    !fib_urpf_check_size (lb[0]->lb_urpf) &&
1460                    ip[0]->dst_address.as_u32 != 0xFFFFFFFF)
1461                   ? IP4_ERROR_SRC_LOOKUP_MISS : error[0]);
1462
1463       error[1] = ((error[1] == IP4_ERROR_UNKNOWN_PROTOCOL &&
1464                    dpo[1]->dpoi_type == DPO_RECEIVE) ?
1465                   IP4_ERROR_SPOOFED_LOCAL_PACKETS : error[1]);
1466       error[1] = ((error[1] == IP4_ERROR_UNKNOWN_PROTOCOL &&
1467                    !fib_urpf_check_size (lb[1]->lb_urpf) &&
1468                    ip[1]->dst_address.as_u32 != 0xFFFFFFFF)
1469                   ? IP4_ERROR_SRC_LOOKUP_MISS : error[1]);
1470
1471       last_check->src.as_u32 = ip[1]->src_address.as_u32;
1472       last_check->lbi = lbi[1];
1473       last_check->error = error[1];
1474     }
1475   else
1476     {
1477       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = last_check->lbi;
1478       vnet_buffer (b[0])->ip.adj_index[VLIB_RX] = last_check->lbi;
1479
1480       vnet_buffer (b[1])->ip.adj_index[VLIB_TX] = last_check->lbi;
1481       vnet_buffer (b[1])->ip.adj_index[VLIB_RX] = last_check->lbi;
1482
1483       error[0] = last_check->error;
1484       error[1] = last_check->error;
1485     }
1486 }
1487
1488 enum ip_local_packet_type_e
1489 {
1490   IP_LOCAL_PACKET_TYPE_L4,
1491   IP_LOCAL_PACKET_TYPE_NAT,
1492   IP_LOCAL_PACKET_TYPE_FRAG,
1493 };
1494
1495 /**
1496  * Determine packet type and next node.
1497  *
1498  * The expectation is that all packets that are not L4 will skip
1499  * checksums and source checks.
1500  */
1501 always_inline u8
1502 ip4_local_classify (vlib_buffer_t * b, ip4_header_t * ip, u16 * next)
1503 {
1504   ip_lookup_main_t *lm = &ip4_main.lookup_main;
1505
1506   if (PREDICT_FALSE (ip4_is_fragment (ip)))
1507     {
1508       *next = IP_LOCAL_NEXT_REASSEMBLY;
1509       return IP_LOCAL_PACKET_TYPE_FRAG;
1510     }
1511   if (PREDICT_FALSE (b->flags & VNET_BUFFER_F_IS_NATED))
1512     {
1513       *next = lm->local_next_by_ip_protocol[ip->protocol];
1514       return IP_LOCAL_PACKET_TYPE_NAT;
1515     }
1516
1517   *next = lm->local_next_by_ip_protocol[ip->protocol];
1518   return IP_LOCAL_PACKET_TYPE_L4;
1519 }
1520
1521 static inline uword
1522 ip4_local_inline (vlib_main_t * vm,
1523                   vlib_node_runtime_t * node,
1524                   vlib_frame_t * frame, int head_of_feature_arc)
1525 {
1526   u32 *from, n_left_from;
1527   vlib_node_runtime_t *error_node =
1528     vlib_node_get_runtime (vm, ip4_input_node.index);
1529   u16 nexts[VLIB_FRAME_SIZE], *next;
1530   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1531   ip4_header_t *ip[2];
1532   u8 error[2], pt[2];
1533
1534   ip4_local_last_check_t last_check = {
1535     .src = {.as_u32 = 0},
1536     .lbi = ~0,
1537     .error = IP4_ERROR_UNKNOWN_PROTOCOL
1538   };
1539
1540   from = vlib_frame_vector_args (frame);
1541   n_left_from = frame->n_vectors;
1542
1543   if (node->flags & VLIB_NODE_FLAG_TRACE)
1544     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1545
1546   vlib_get_buffers (vm, from, bufs, n_left_from);
1547   b = bufs;
1548   next = nexts;
1549
1550   while (n_left_from >= 6)
1551     {
1552       u8 not_batch = 0;
1553
1554       /* Prefetch next iteration. */
1555       {
1556         vlib_prefetch_buffer_header (b[4], LOAD);
1557         vlib_prefetch_buffer_header (b[5], LOAD);
1558
1559         CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD);
1560         CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD);
1561       }
1562
1563       error[0] = error[1] = IP4_ERROR_UNKNOWN_PROTOCOL;
1564
1565       ip[0] = vlib_buffer_get_current (b[0]);
1566       ip[1] = vlib_buffer_get_current (b[1]);
1567
1568       vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data;
1569       vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data;
1570
1571       pt[0] = ip4_local_classify (b[0], ip[0], &next[0]);
1572       pt[1] = ip4_local_classify (b[1], ip[1], &next[1]);
1573
1574       not_batch = pt[0] ^ pt[1];
1575
1576       if (head_of_feature_arc == 0 || (pt[0] && not_batch == 0))
1577         goto skip_checks;
1578
1579       if (PREDICT_TRUE (not_batch == 0))
1580         {
1581           ip4_local_check_l4_csum_x2 (vm, b, ip, error);
1582           ip4_local_check_src_x2 (b, ip, &last_check, error);
1583         }
1584       else
1585         {
1586           if (!pt[0])
1587             {
1588               ip4_local_check_l4_csum (vm, b[0], ip[0], &error[0]);
1589               ip4_local_check_src (b[0], ip[0], &last_check, &error[0]);
1590             }
1591           if (!pt[1])
1592             {
1593               ip4_local_check_l4_csum (vm, b[1], ip[1], &error[1]);
1594               ip4_local_check_src (b[1], ip[1], &last_check, &error[1]);
1595             }
1596         }
1597
1598     skip_checks:
1599
1600       ip4_local_set_next_and_error (error_node, b[0], &next[0], error[0],
1601                                     head_of_feature_arc);
1602       ip4_local_set_next_and_error (error_node, b[1], &next[1], error[1],
1603                                     head_of_feature_arc);
1604
1605       b += 2;
1606       next += 2;
1607       n_left_from -= 2;
1608     }
1609
1610   while (n_left_from > 0)
1611     {
1612       error[0] = IP4_ERROR_UNKNOWN_PROTOCOL;
1613
1614       ip[0] = vlib_buffer_get_current (b[0]);
1615       vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data;
1616       pt[0] = ip4_local_classify (b[0], ip[0], &next[0]);
1617
1618       if (head_of_feature_arc == 0 || pt[0])
1619         goto skip_check;
1620
1621       ip4_local_check_l4_csum (vm, b[0], ip[0], &error[0]);
1622       ip4_local_check_src (b[0], ip[0], &last_check, &error[0]);
1623
1624     skip_check:
1625
1626       ip4_local_set_next_and_error (error_node, b[0], &next[0], error[0],
1627                                     head_of_feature_arc);
1628
1629       b += 1;
1630       next += 1;
1631       n_left_from -= 1;
1632     }
1633
1634   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
1635   return frame->n_vectors;
1636 }
1637
1638 VLIB_NODE_FN (ip4_local_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
1639                                vlib_frame_t * frame)
1640 {
1641   return ip4_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1642 }
1643
1644 /* *INDENT-OFF* */
1645 VLIB_REGISTER_NODE (ip4_local_node) =
1646 {
1647   .name = "ip4-local",
1648   .vector_size = sizeof (u32),
1649   .format_trace = format_ip4_forward_next_trace,
1650   .n_next_nodes = IP_LOCAL_N_NEXT,
1651   .next_nodes =
1652   {
1653     [IP_LOCAL_NEXT_DROP] = "ip4-drop",
1654     [IP_LOCAL_NEXT_PUNT] = "ip4-punt",
1655     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
1656     [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",
1657     [IP_LOCAL_NEXT_REASSEMBLY] = "ip4-reassembly",
1658   },
1659 };
1660 /* *INDENT-ON* */
1661
1662
1663 VLIB_NODE_FN (ip4_local_end_of_arc_node) (vlib_main_t * vm,
1664                                           vlib_node_runtime_t * node,
1665                                           vlib_frame_t * frame)
1666 {
1667   return ip4_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1668 }
1669
1670 /* *INDENT-OFF* */
1671 VLIB_REGISTER_NODE (ip4_local_end_of_arc_node) = {
1672   .name = "ip4-local-end-of-arc",
1673   .vector_size = sizeof (u32),
1674
1675   .format_trace = format_ip4_forward_next_trace,
1676   .sibling_of = "ip4-local",
1677 };
1678
1679 VNET_FEATURE_INIT (ip4_local_end_of_arc, static) = {
1680   .arc_name = "ip4-local",
1681   .node_name = "ip4-local-end-of-arc",
1682   .runs_before = 0, /* not before any other features */
1683 };
1684 /* *INDENT-ON* */
1685
1686 #ifndef CLIB_MARCH_VARIANT
1687 void
1688 ip4_register_protocol (u32 protocol, u32 node_index)
1689 {
1690   vlib_main_t *vm = vlib_get_main ();
1691   ip4_main_t *im = &ip4_main;
1692   ip_lookup_main_t *lm = &im->lookup_main;
1693
1694   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1695   lm->local_next_by_ip_protocol[protocol] =
1696     vlib_node_add_next (vm, ip4_local_node.index, node_index);
1697 }
1698 #endif
1699
1700 static clib_error_t *
1701 show_ip_local_command_fn (vlib_main_t * vm,
1702                           unformat_input_t * input, vlib_cli_command_t * cmd)
1703 {
1704   ip4_main_t *im = &ip4_main;
1705   ip_lookup_main_t *lm = &im->lookup_main;
1706   int i;
1707
1708   vlib_cli_output (vm, "Protocols handled by ip4_local");
1709   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
1710     {
1711       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
1712         {
1713           u32 node_index = vlib_get_node (vm,
1714                                           ip4_local_node.index)->
1715             next_nodes[lm->local_next_by_ip_protocol[i]];
1716           vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm,
1717                            node_index);
1718         }
1719     }
1720   return 0;
1721 }
1722
1723
1724
1725 /*?
1726  * Display the set of protocols handled by the local IPv4 stack.
1727  *
1728  * @cliexpar
1729  * Example of how to display local protocol table:
1730  * @cliexstart{show ip local}
1731  * Protocols handled by ip4_local
1732  * 1
1733  * 17
1734  * 47
1735  * @cliexend
1736 ?*/
1737 /* *INDENT-OFF* */
1738 VLIB_CLI_COMMAND (show_ip_local, static) =
1739 {
1740   .path = "show ip local",
1741   .function = show_ip_local_command_fn,
1742   .short_help = "show ip local",
1743 };
1744 /* *INDENT-ON* */
1745
1746 always_inline uword
1747 ip4_arp_inline (vlib_main_t * vm,
1748                 vlib_node_runtime_t * node,
1749                 vlib_frame_t * frame, int is_glean)
1750 {
1751   vnet_main_t *vnm = vnet_get_main ();
1752   ip4_main_t *im = &ip4_main;
1753   ip_lookup_main_t *lm = &im->lookup_main;
1754   u32 *from, *to_next_drop;
1755   uword n_left_from, n_left_to_next_drop, next_index;
1756   u32 thread_index = vm->thread_index;
1757   u64 seed;
1758
1759   if (node->flags & VLIB_NODE_FLAG_TRACE)
1760     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1761
1762   seed = throttle_seed (&im->arp_throttle, thread_index, vlib_time_now (vm));
1763
1764   from = vlib_frame_vector_args (frame);
1765   n_left_from = frame->n_vectors;
1766   next_index = node->cached_next_index;
1767   if (next_index == IP4_ARP_NEXT_DROP)
1768     next_index = IP4_ARP_N_NEXT;        /* point to first interface */
1769
1770   while (n_left_from > 0)
1771     {
1772       vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
1773                            to_next_drop, n_left_to_next_drop);
1774
1775       while (n_left_from > 0 && n_left_to_next_drop > 0)
1776         {
1777           u32 pi0, bi0, adj_index0, sw_if_index0;
1778           ip_adjacency_t *adj0;
1779           vlib_buffer_t *p0, *b0;
1780           ip4_address_t resolve0;
1781           ethernet_arp_header_t *h0;
1782           vnet_hw_interface_t *hw_if0;
1783           u64 r0;
1784
1785           pi0 = from[0];
1786           p0 = vlib_get_buffer (vm, pi0);
1787
1788           from += 1;
1789           n_left_from -= 1;
1790           to_next_drop[0] = pi0;
1791           to_next_drop += 1;
1792           n_left_to_next_drop -= 1;
1793
1794           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1795           adj0 = adj_get (adj_index0);
1796
1797           if (is_glean)
1798             {
1799               /* resolve the packet's destination */
1800               ip4_header_t *ip0 = vlib_buffer_get_current (p0);
1801               resolve0 = ip0->dst_address;
1802             }
1803           else
1804             {
1805               /* resolve the incomplete adj */
1806               resolve0 = adj0->sub_type.nbr.next_hop.ip4;
1807             }
1808
1809           /* combine the address and interface for the hash key */
1810           sw_if_index0 = adj0->rewrite_header.sw_if_index;
1811           r0 = (u64) resolve0.data_u32 << 32;
1812           r0 |= sw_if_index0;
1813
1814           if (throttle_check (&im->arp_throttle, thread_index, r0, seed))
1815             {
1816               p0->error = node->errors[IP4_ARP_ERROR_THROTTLED];
1817               continue;
1818             }
1819
1820           /*
1821            * the adj has been updated to a rewrite but the node the DPO that got
1822            * us here hasn't - yet. no big deal. we'll drop while we wait.
1823            */
1824           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
1825             {
1826               p0->error = node->errors[IP4_ARP_ERROR_RESOLVED];
1827               continue;
1828             }
1829
1830           /*
1831            * Can happen if the control-plane is programming tables
1832            * with traffic flowing; at least that's today's lame excuse.
1833            */
1834           if ((is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_GLEAN)
1835               || (!is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP))
1836             {
1837               p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
1838               continue;
1839             }
1840           /* Send ARP request. */
1841           h0 =
1842             vlib_packet_template_get_packet (vm,
1843                                              &im->ip4_arp_request_packet_template,
1844                                              &bi0);
1845
1846           /* Seems we're out of buffers */
1847           if (PREDICT_FALSE (!h0))
1848             {
1849               p0->error = node->errors[IP4_ARP_ERROR_NO_BUFFERS];
1850               continue;
1851             }
1852
1853           /* Add rewrite/encap string for ARP packet. */
1854           vnet_rewrite_one_header (adj0[0], h0, sizeof (ethernet_header_t));
1855
1856           hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1857
1858           /* Src ethernet address in ARP header. */
1859           clib_memcpy_fast (h0->ip4_over_ethernet[0].ethernet,
1860                             hw_if0->hw_address,
1861                             sizeof (h0->ip4_over_ethernet[0].ethernet));
1862           if (is_glean)
1863             {
1864               /* The interface's source address is stashed in the Glean Adj */
1865               h0->ip4_over_ethernet[0].ip4 =
1866                 adj0->sub_type.glean.receive_addr.ip4;
1867             }
1868           else
1869             {
1870               /* Src IP address in ARP header. */
1871               if (ip4_src_address_for_packet (lm, sw_if_index0,
1872                                               &h0->ip4_over_ethernet[0].ip4))
1873                 {
1874                   /* No source address available */
1875                   p0->error = node->errors[IP4_ARP_ERROR_NO_SOURCE_ADDRESS];
1876                   vlib_buffer_free (vm, &bi0, 1);
1877                   continue;
1878                 }
1879             }
1880           h0->ip4_over_ethernet[1].ip4 = resolve0;
1881
1882           p0->error = node->errors[IP4_ARP_ERROR_REQUEST_SENT];
1883
1884           vlib_buffer_copy_trace_flag (vm, p0, bi0);
1885           b0 = vlib_get_buffer (vm, bi0);
1886           VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
1887           vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
1888
1889           vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
1890
1891           vlib_set_next_frame_buffer (vm, node,
1892                                       adj0->rewrite_header.next_index, bi0);
1893         }
1894
1895       vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
1896     }
1897
1898   return frame->n_vectors;
1899 }
1900
1901 VLIB_NODE_FN (ip4_arp_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
1902                              vlib_frame_t * frame)
1903 {
1904   return (ip4_arp_inline (vm, node, frame, 0));
1905 }
1906
1907 VLIB_NODE_FN (ip4_glean_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
1908                                vlib_frame_t * frame)
1909 {
1910   return (ip4_arp_inline (vm, node, frame, 1));
1911 }
1912
1913 static char *ip4_arp_error_strings[] = {
1914   [IP4_ARP_ERROR_THROTTLED] = "ARP requests throttled",
1915   [IP4_ARP_ERROR_RESOLVED] = "ARP requests resolved",
1916   [IP4_ARP_ERROR_NO_BUFFERS] = "ARP requests out of buffer",
1917   [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
1918   [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
1919   [IP4_ARP_ERROR_NO_SOURCE_ADDRESS] = "no source address for ARP request",
1920 };
1921
1922 /* *INDENT-OFF* */
1923 VLIB_REGISTER_NODE (ip4_arp_node) =
1924 {
1925   .name = "ip4-arp",
1926   .vector_size = sizeof (u32),
1927   .format_trace = format_ip4_forward_next_trace,
1928   .n_errors = ARRAY_LEN (ip4_arp_error_strings),
1929   .error_strings = ip4_arp_error_strings,
1930   .n_next_nodes = IP4_ARP_N_NEXT,
1931   .next_nodes =
1932   {
1933     [IP4_ARP_NEXT_DROP] = "error-drop",
1934   },
1935 };
1936
1937 VLIB_REGISTER_NODE (ip4_glean_node) =
1938 {
1939   .name = "ip4-glean",
1940   .vector_size = sizeof (u32),
1941   .format_trace = format_ip4_forward_next_trace,
1942   .n_errors = ARRAY_LEN (ip4_arp_error_strings),
1943   .error_strings = ip4_arp_error_strings,
1944   .n_next_nodes = IP4_ARP_N_NEXT,
1945   .next_nodes = {
1946   [IP4_ARP_NEXT_DROP] = "error-drop",
1947   },
1948 };
1949 /* *INDENT-ON* */
1950
1951 #define foreach_notrace_ip4_arp_error           \
1952 _(THROTTLED)                                    \
1953 _(RESOLVED)                                     \
1954 _(NO_BUFFERS)                                   \
1955 _(REQUEST_SENT)                                 \
1956 _(NON_ARP_ADJ)                                  \
1957 _(NO_SOURCE_ADDRESS)
1958
1959 static clib_error_t *
1960 arp_notrace_init (vlib_main_t * vm)
1961 {
1962   vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, ip4_arp_node.index);
1963
1964   /* don't trace ARP request packets */
1965 #define _(a)                                    \
1966     vnet_pcap_drop_trace_filter_add_del         \
1967         (rt->errors[IP4_ARP_ERROR_##a],         \
1968          1 /* is_add */);
1969   foreach_notrace_ip4_arp_error;
1970 #undef _
1971   return 0;
1972 }
1973
1974 VLIB_INIT_FUNCTION (arp_notrace_init);
1975
1976
1977 #ifndef CLIB_MARCH_VARIANT
1978 /* Send an ARP request to see if given destination is reachable on given interface. */
1979 clib_error_t *
1980 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index,
1981                     u8 refresh)
1982 {
1983   vnet_main_t *vnm = vnet_get_main ();
1984   ip4_main_t *im = &ip4_main;
1985   ethernet_arp_header_t *h;
1986   ip4_address_t *src;
1987   ip_interface_address_t *ia;
1988   ip_adjacency_t *adj;
1989   vnet_hw_interface_t *hi;
1990   vnet_sw_interface_t *si;
1991   vlib_buffer_t *b;
1992   adj_index_t ai;
1993   u32 bi = 0;
1994   u8 unicast_rewrite = 0;
1995
1996   si = vnet_get_sw_interface (vnm, sw_if_index);
1997
1998   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
1999     {
2000       return clib_error_return (0, "%U: interface %U down",
2001                                 format_ip4_address, dst,
2002                                 format_vnet_sw_if_index_name, vnm,
2003                                 sw_if_index);
2004     }
2005
2006   src =
2007     ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2008   if (!src)
2009     {
2010       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2011       return clib_error_return
2012         (0,
2013          "no matching interface address for destination %U (interface %U)",
2014          format_ip4_address, dst, format_vnet_sw_if_index_name, vnm,
2015          sw_if_index);
2016     }
2017
2018   h = vlib_packet_template_get_packet (vm,
2019                                        &im->ip4_arp_request_packet_template,
2020                                        &bi);
2021
2022   if (!h)
2023     return clib_error_return (0, "ARP request packet allocation failed");
2024
2025   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2026   if (PREDICT_FALSE (!hi->hw_address))
2027     {
2028       return clib_error_return (0, "%U: interface %U do not support ip probe",
2029                                 format_ip4_address, dst,
2030                                 format_vnet_sw_if_index_name, vnm,
2031                                 sw_if_index);
2032     }
2033
2034   clib_memcpy_fast (h->ip4_over_ethernet[0].ethernet, hi->hw_address,
2035                     sizeof (h->ip4_over_ethernet[0].ethernet));
2036
2037   h->ip4_over_ethernet[0].ip4 = src[0];
2038   h->ip4_over_ethernet[1].ip4 = dst[0];
2039
2040   b = vlib_get_buffer (vm, bi);
2041   vnet_buffer (b)->sw_if_index[VLIB_RX] =
2042     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2043
2044   ip46_address_t nh = {
2045     .ip4 = *dst,
2046   };
2047
2048   ai = adj_nbr_add_or_lock (FIB_PROTOCOL_IP4,
2049                             VNET_LINK_IP4, &nh, sw_if_index);
2050   adj = adj_get (ai);
2051
2052   /* Peer has been previously resolved, retrieve glean adj instead */
2053   if (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE)
2054     {
2055       if (refresh)
2056         unicast_rewrite = 1;
2057       else
2058         {
2059           adj_unlock (ai);
2060           ai = adj_glean_add_or_lock (FIB_PROTOCOL_IP4,
2061                                       VNET_LINK_IP4, sw_if_index, &nh);
2062           adj = adj_get (ai);
2063         }
2064     }
2065
2066   /* Add encapsulation string for software interface (e.g. ethernet header). */
2067   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2068   if (unicast_rewrite)
2069     {
2070       u16 *etype = vlib_buffer_get_current (b) - 2;
2071       etype[0] = clib_host_to_net_u16 (ETHERNET_TYPE_ARP);
2072     }
2073   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2074
2075   {
2076     vlib_frame_t *f = vlib_get_frame_to_node (vm, hi->output_node_index);
2077     u32 *to_next = vlib_frame_vector_args (f);
2078     to_next[0] = bi;
2079     f->n_vectors = 1;
2080     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2081   }
2082
2083   adj_unlock (ai);
2084   return /* no error */ 0;
2085 }
2086 #endif
2087
2088 typedef enum
2089 {
2090   IP4_REWRITE_NEXT_DROP,
2091   IP4_REWRITE_NEXT_ICMP_ERROR,
2092   IP4_REWRITE_NEXT_FRAGMENT,
2093   IP4_REWRITE_N_NEXT            /* Last */
2094 } ip4_rewrite_next_t;
2095
2096 /**
2097  * This bits of an IPv4 address to mask to construct a multicast
2098  * MAC address
2099  */
2100 #if CLIB_ARCH_IS_BIG_ENDIAN
2101 #define IP4_MCAST_ADDR_MASK 0x007fffff
2102 #else
2103 #define IP4_MCAST_ADDR_MASK 0xffff7f00
2104 #endif
2105
2106 always_inline void
2107 ip4_mtu_check (vlib_buffer_t * b, u16 packet_len,
2108                u16 adj_packet_bytes, bool df, u16 * next, u32 * error)
2109 {
2110   if (packet_len > adj_packet_bytes)
2111     {
2112       *error = IP4_ERROR_MTU_EXCEEDED;
2113       if (df)
2114         {
2115           icmp4_error_set_vnet_buffer
2116             (b, ICMP4_destination_unreachable,
2117              ICMP4_destination_unreachable_fragmentation_needed_and_dont_fragment_set,
2118              adj_packet_bytes);
2119           *next = IP4_REWRITE_NEXT_ICMP_ERROR;
2120         }
2121       else
2122         {
2123           /* IP fragmentation */
2124           ip_frag_set_vnet_buffer (b, adj_packet_bytes,
2125                                    IP4_FRAG_NEXT_IP4_REWRITE, 0);
2126           *next = IP4_REWRITE_NEXT_FRAGMENT;
2127         }
2128     }
2129 }
2130
2131 /* Decrement TTL & update checksum.
2132    Works either endian, so no need for byte swap. */
2133 static_always_inline void
2134 ip4_ttl_and_checksum_check (vlib_buffer_t * b, ip4_header_t * ip, u16 * next,
2135                             u32 * error)
2136 {
2137   i32 ttl;
2138   u32 checksum;
2139   if (PREDICT_FALSE (b->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED))
2140     {
2141       b->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2142       return;
2143     }
2144
2145   ttl = ip->ttl;
2146
2147   /* Input node should have reject packets with ttl 0. */
2148   ASSERT (ip->ttl > 0);
2149
2150   checksum = ip->checksum + clib_host_to_net_u16 (0x0100);
2151   checksum += checksum >= 0xffff;
2152
2153   ip->checksum = checksum;
2154   ttl -= 1;
2155   ip->ttl = ttl;
2156
2157   /*
2158    * If the ttl drops below 1 when forwarding, generate
2159    * an ICMP response.
2160    */
2161   if (PREDICT_FALSE (ttl <= 0))
2162     {
2163       *error = IP4_ERROR_TIME_EXPIRED;
2164       vnet_buffer (b)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2165       icmp4_error_set_vnet_buffer (b, ICMP4_time_exceeded,
2166                                    ICMP4_time_exceeded_ttl_exceeded_in_transit,
2167                                    0);
2168       *next = IP4_REWRITE_NEXT_ICMP_ERROR;
2169     }
2170
2171   /* Verify checksum. */
2172   ASSERT ((ip->checksum == ip4_header_checksum (ip)) ||
2173           (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
2174 }
2175
2176
2177 always_inline uword
2178 ip4_rewrite_inline (vlib_main_t * vm,
2179                     vlib_node_runtime_t * node,
2180                     vlib_frame_t * frame,
2181                     int do_counters, int is_midchain, int is_mcast)
2182 {
2183   ip_lookup_main_t *lm = &ip4_main.lookup_main;
2184   u32 *from = vlib_frame_vector_args (frame);
2185   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
2186   u16 nexts[VLIB_FRAME_SIZE], *next;
2187   u32 n_left_from;
2188   vlib_node_runtime_t *error_node =
2189     vlib_node_get_runtime (vm, ip4_input_node.index);
2190
2191   n_left_from = frame->n_vectors;
2192   u32 thread_index = vm->thread_index;
2193
2194   vlib_get_buffers (vm, from, bufs, n_left_from);
2195   clib_memset_u16 (nexts, IP4_REWRITE_NEXT_DROP, n_left_from);
2196
2197   if (n_left_from >= 6)
2198     {
2199       int i;
2200       for (i = 2; i < 6; i++)
2201         vlib_prefetch_buffer_header (bufs[i], LOAD);
2202     }
2203
2204   next = nexts;
2205   b = bufs;
2206   while (n_left_from >= 8)
2207     {
2208       ip_adjacency_t *adj0, *adj1;
2209       ip4_header_t *ip0, *ip1;
2210       u32 rw_len0, error0, adj_index0;
2211       u32 rw_len1, error1, adj_index1;
2212       u32 tx_sw_if_index0, tx_sw_if_index1;
2213       u8 *p;
2214
2215       vlib_prefetch_buffer_header (b[6], LOAD);
2216       vlib_prefetch_buffer_header (b[7], LOAD);
2217
2218       adj_index0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
2219       adj_index1 = vnet_buffer (b[1])->ip.adj_index[VLIB_TX];
2220
2221       /*
2222        * pre-fetch the per-adjacency counters
2223        */
2224       if (do_counters)
2225         {
2226           vlib_prefetch_combined_counter (&adjacency_counters,
2227                                           thread_index, adj_index0);
2228           vlib_prefetch_combined_counter (&adjacency_counters,
2229                                           thread_index, adj_index1);
2230         }
2231
2232       ip0 = vlib_buffer_get_current (b[0]);
2233       ip1 = vlib_buffer_get_current (b[1]);
2234
2235       error0 = error1 = IP4_ERROR_NONE;
2236
2237       ip4_ttl_and_checksum_check (b[0], ip0, next + 0, &error0);
2238       ip4_ttl_and_checksum_check (b[1], ip1, next + 1, &error1);
2239
2240       /* Rewrite packet header and updates lengths. */
2241       adj0 = adj_get (adj_index0);
2242       adj1 = adj_get (adj_index1);
2243
2244       /* Worth pipelining. No guarantee that adj0,1 are hot... */
2245       rw_len0 = adj0[0].rewrite_header.data_bytes;
2246       rw_len1 = adj1[0].rewrite_header.data_bytes;
2247       vnet_buffer (b[0])->ip.save_rewrite_length = rw_len0;
2248       vnet_buffer (b[1])->ip.save_rewrite_length = rw_len1;
2249
2250       p = vlib_buffer_get_current (b[2]);
2251       CLIB_PREFETCH (p - CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES, STORE);
2252       CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
2253
2254       p = vlib_buffer_get_current (b[3]);
2255       CLIB_PREFETCH (p - CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES, STORE);
2256       CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
2257
2258       /* Check MTU of outgoing interface. */
2259       ip4_mtu_check (b[0], clib_net_to_host_u16 (ip0->length),
2260                      adj0[0].rewrite_header.max_l3_packet_bytes,
2261                      ip0->flags_and_fragment_offset &
2262                      clib_host_to_net_u16 (IP4_HEADER_FLAG_DONT_FRAGMENT),
2263                      next + 0, &error0);
2264       ip4_mtu_check (b[1], clib_net_to_host_u16 (ip1->length),
2265                      adj1[0].rewrite_header.max_l3_packet_bytes,
2266                      ip1->flags_and_fragment_offset &
2267                      clib_host_to_net_u16 (IP4_HEADER_FLAG_DONT_FRAGMENT),
2268                      next + 1, &error1);
2269
2270       if (is_mcast)
2271         {
2272           error0 = ((adj0[0].rewrite_header.sw_if_index ==
2273                      vnet_buffer (b[0])->sw_if_index[VLIB_RX]) ?
2274                     IP4_ERROR_SAME_INTERFACE : error0);
2275           error1 = ((adj1[0].rewrite_header.sw_if_index ==
2276                      vnet_buffer (b[1])->sw_if_index[VLIB_RX]) ?
2277                     IP4_ERROR_SAME_INTERFACE : error1);
2278         }
2279
2280       b[0]->error = error_node->errors[error0];
2281       b[1]->error = error_node->errors[error1];
2282       /* Don't adjust the buffer for ttl issue; icmp-error node wants
2283        * to see the IP headerr */
2284       if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2285         {
2286           u32 next_index = adj0[0].rewrite_header.next_index;
2287           b[0]->current_data -= rw_len0;
2288           b[0]->current_length += rw_len0;
2289           tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2290           vnet_buffer (b[0])->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2291
2292           if (PREDICT_FALSE
2293               (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2294             vnet_feature_arc_start (lm->output_feature_arc_index,
2295                                     tx_sw_if_index0, &next_index, b[0]);
2296           next[0] = next_index;
2297         }
2298       if (PREDICT_TRUE (error1 == IP4_ERROR_NONE))
2299         {
2300           u32 next_index = adj1[0].rewrite_header.next_index;
2301           b[1]->current_data -= rw_len1;
2302           b[1]->current_length += rw_len1;
2303
2304           tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
2305           vnet_buffer (b[1])->sw_if_index[VLIB_TX] = tx_sw_if_index1;
2306
2307           if (PREDICT_FALSE
2308               (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2309             vnet_feature_arc_start (lm->output_feature_arc_index,
2310                                     tx_sw_if_index1, &next_index, b[1]);
2311           next[1] = next_index;
2312         }
2313
2314       /* Guess we are only writing on simple Ethernet header. */
2315       vnet_rewrite_two_headers (adj0[0], adj1[0],
2316                                 ip0, ip1, sizeof (ethernet_header_t));
2317
2318       /*
2319        * Bump the per-adjacency counters
2320        */
2321       if (do_counters)
2322         {
2323           vlib_increment_combined_counter
2324             (&adjacency_counters,
2325              thread_index,
2326              adj_index0, 1, vlib_buffer_length_in_chain (vm, b[0]) + rw_len0);
2327
2328           vlib_increment_combined_counter
2329             (&adjacency_counters,
2330              thread_index,
2331              adj_index1, 1, vlib_buffer_length_in_chain (vm, b[1]) + rw_len1);
2332         }
2333
2334       if (is_midchain)
2335         {
2336           adj0->sub_type.midchain.fixup_func
2337             (vm, adj0, b[0], adj0->sub_type.midchain.fixup_data);
2338           adj1->sub_type.midchain.fixup_func
2339             (vm, adj1, b[1], adj0->sub_type.midchain.fixup_data);
2340         }
2341
2342       if (is_mcast)
2343         {
2344           /*
2345            * copy bytes from the IP address into the MAC rewrite
2346            */
2347           vnet_ip_mcast_fixup_header (IP4_MCAST_ADDR_MASK,
2348                                       adj0->rewrite_header.dst_mcast_offset,
2349                                       &ip0->dst_address.as_u32, (u8 *) ip0);
2350           vnet_ip_mcast_fixup_header (IP4_MCAST_ADDR_MASK,
2351                                       adj0->rewrite_header.dst_mcast_offset,
2352                                       &ip1->dst_address.as_u32, (u8 *) ip1);
2353         }
2354
2355       next += 2;
2356       b += 2;
2357       n_left_from -= 2;
2358     }
2359
2360   while (n_left_from > 0)
2361     {
2362       ip_adjacency_t *adj0;
2363       ip4_header_t *ip0;
2364       u32 rw_len0, adj_index0, error0;
2365       u32 tx_sw_if_index0;
2366
2367       adj_index0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
2368
2369       adj0 = adj_get (adj_index0);
2370
2371       if (do_counters)
2372         vlib_prefetch_combined_counter (&adjacency_counters,
2373                                         thread_index, adj_index0);
2374
2375       ip0 = vlib_buffer_get_current (b[0]);
2376
2377       error0 = IP4_ERROR_NONE;
2378
2379       ip4_ttl_and_checksum_check (b[0], ip0, next + 0, &error0);
2380
2381
2382       /* Update packet buffer attributes/set output interface. */
2383       rw_len0 = adj0[0].rewrite_header.data_bytes;
2384       vnet_buffer (b[0])->ip.save_rewrite_length = rw_len0;
2385
2386       /* Check MTU of outgoing interface. */
2387       ip4_mtu_check (b[0], clib_net_to_host_u16 (ip0->length),
2388                      adj0[0].rewrite_header.max_l3_packet_bytes,
2389                      ip0->flags_and_fragment_offset &
2390                      clib_host_to_net_u16 (IP4_HEADER_FLAG_DONT_FRAGMENT),
2391                      next + 0, &error0);
2392
2393       if (is_mcast)
2394         {
2395           error0 = ((adj0[0].rewrite_header.sw_if_index ==
2396                      vnet_buffer (b[0])->sw_if_index[VLIB_RX]) ?
2397                     IP4_ERROR_SAME_INTERFACE : error0);
2398         }
2399       b[0]->error = error_node->errors[error0];
2400
2401       /* Don't adjust the buffer for ttl issue; icmp-error node wants
2402        * to see the IP headerr */
2403       if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2404         {
2405           u32 next_index = adj0[0].rewrite_header.next_index;
2406           b[0]->current_data -= rw_len0;
2407           b[0]->current_length += rw_len0;
2408           tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2409           vnet_buffer (b[0])->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2410
2411           if (PREDICT_FALSE
2412               (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2413             vnet_feature_arc_start (lm->output_feature_arc_index,
2414                                     tx_sw_if_index0, &next_index, b[0]);
2415           next[0] = next_index;
2416         }
2417
2418       /* Guess we are only writing on simple Ethernet header. */
2419       vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2420
2421       if (do_counters)
2422         vlib_increment_combined_counter
2423           (&adjacency_counters,
2424            thread_index, adj_index0, 1,
2425            vlib_buffer_length_in_chain (vm, b[0]) + rw_len0);
2426
2427       if (is_midchain)
2428         {
2429           adj0->sub_type.midchain.fixup_func
2430             (vm, adj0, b[0], adj0->sub_type.midchain.fixup_data);
2431         }
2432
2433       if (is_mcast)
2434         {
2435           /*
2436            * copy bytes from the IP address into the MAC rewrite
2437            */
2438           vnet_ip_mcast_fixup_header (IP4_MCAST_ADDR_MASK,
2439                                       adj0->rewrite_header.dst_mcast_offset,
2440                                       &ip0->dst_address.as_u32, (u8 *) ip0);
2441         }
2442
2443       next += 1;
2444       b += 1;
2445       n_left_from -= 1;
2446     }
2447
2448
2449   /* Need to do trace after rewrites to pick up new packet data. */
2450   if (node->flags & VLIB_NODE_FLAG_TRACE)
2451     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2452
2453   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
2454   return frame->n_vectors;
2455 }
2456
2457
2458 /** @brief IPv4 rewrite node.
2459     @node ip4-rewrite
2460
2461     This is the IPv4 transit-rewrite node: decrement TTL, fix the ipv4
2462     header checksum, fetch the ip adjacency, check the outbound mtu,
2463     apply the adjacency rewrite, and send pkts to the adjacency
2464     rewrite header's rewrite_next_index.
2465
2466     @param vm vlib_main_t corresponding to the current thread
2467     @param node vlib_node_runtime_t
2468     @param frame vlib_frame_t whose contents should be dispatched
2469
2470     @par Graph mechanics: buffer metadata, next index usage
2471
2472     @em Uses:
2473     - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
2474         - the rewrite adjacency index
2475     - <code>adj->lookup_next_index</code>
2476         - Must be IP_LOOKUP_NEXT_REWRITE or IP_LOOKUP_NEXT_ARP, otherwise
2477           the packet will be dropped.
2478     - <code>adj->rewrite_header</code>
2479         - Rewrite string length, rewrite string, next_index
2480
2481     @em Sets:
2482     - <code>b->current_data, b->current_length</code>
2483         - Updated net of applying the rewrite string
2484
2485     <em>Next Indices:</em>
2486     - <code> adj->rewrite_header.next_index </code>
2487       or @c ip4-drop
2488 */
2489
2490 VLIB_NODE_FN (ip4_rewrite_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
2491                                  vlib_frame_t * frame)
2492 {
2493   if (adj_are_counters_enabled ())
2494     return ip4_rewrite_inline (vm, node, frame, 1, 0, 0);
2495   else
2496     return ip4_rewrite_inline (vm, node, frame, 0, 0, 0);
2497 }
2498
2499 VLIB_NODE_FN (ip4_rewrite_bcast_node) (vlib_main_t * vm,
2500                                        vlib_node_runtime_t * node,
2501                                        vlib_frame_t * frame)
2502 {
2503   if (adj_are_counters_enabled ())
2504     return ip4_rewrite_inline (vm, node, frame, 1, 0, 0);
2505   else
2506     return ip4_rewrite_inline (vm, node, frame, 0, 0, 0);
2507 }
2508
2509 VLIB_NODE_FN (ip4_midchain_node) (vlib_main_t * vm,
2510                                   vlib_node_runtime_t * node,
2511                                   vlib_frame_t * frame)
2512 {
2513   if (adj_are_counters_enabled ())
2514     return ip4_rewrite_inline (vm, node, frame, 1, 1, 0);
2515   else
2516     return ip4_rewrite_inline (vm, node, frame, 0, 1, 0);
2517 }
2518
2519 VLIB_NODE_FN (ip4_rewrite_mcast_node) (vlib_main_t * vm,
2520                                        vlib_node_runtime_t * node,
2521                                        vlib_frame_t * frame)
2522 {
2523   if (adj_are_counters_enabled ())
2524     return ip4_rewrite_inline (vm, node, frame, 1, 0, 1);
2525   else
2526     return ip4_rewrite_inline (vm, node, frame, 0, 0, 1);
2527 }
2528
2529 VLIB_NODE_FN (ip4_mcast_midchain_node) (vlib_main_t * vm,
2530                                         vlib_node_runtime_t * node,
2531                                         vlib_frame_t * frame)
2532 {
2533   if (adj_are_counters_enabled ())
2534     return ip4_rewrite_inline (vm, node, frame, 1, 1, 1);
2535   else
2536     return ip4_rewrite_inline (vm, node, frame, 0, 1, 1);
2537 }
2538
2539 /* *INDENT-OFF* */
2540 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2541   .name = "ip4-rewrite",
2542   .vector_size = sizeof (u32),
2543
2544   .format_trace = format_ip4_rewrite_trace,
2545
2546   .n_next_nodes = IP4_REWRITE_N_NEXT,
2547   .next_nodes = {
2548     [IP4_REWRITE_NEXT_DROP] = "ip4-drop",
2549     [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",
2550     [IP4_REWRITE_NEXT_FRAGMENT] = "ip4-frag",
2551   },
2552 };
2553
2554 VLIB_REGISTER_NODE (ip4_rewrite_bcast_node) = {
2555   .name = "ip4-rewrite-bcast",
2556   .vector_size = sizeof (u32),
2557
2558   .format_trace = format_ip4_rewrite_trace,
2559   .sibling_of = "ip4-rewrite",
2560 };
2561
2562 VLIB_REGISTER_NODE (ip4_rewrite_mcast_node) = {
2563   .name = "ip4-rewrite-mcast",
2564   .vector_size = sizeof (u32),
2565
2566   .format_trace = format_ip4_rewrite_trace,
2567   .sibling_of = "ip4-rewrite",
2568 };
2569
2570 VLIB_REGISTER_NODE (ip4_mcast_midchain_node) = {
2571   .name = "ip4-mcast-midchain",
2572   .vector_size = sizeof (u32),
2573
2574   .format_trace = format_ip4_rewrite_trace,
2575   .sibling_of = "ip4-rewrite",
2576 };
2577
2578 VLIB_REGISTER_NODE (ip4_midchain_node) = {
2579   .name = "ip4-midchain",
2580   .vector_size = sizeof (u32),
2581   .format_trace = format_ip4_forward_next_trace,
2582   .sibling_of =  "ip4-rewrite",
2583 };
2584 /* *INDENT-ON */
2585
2586 static int
2587 ip4_lookup_validate (ip4_address_t * a, u32 fib_index0)
2588 {
2589   ip4_fib_mtrie_t *mtrie0;
2590   ip4_fib_mtrie_leaf_t leaf0;
2591   u32 lbi0;
2592
2593   mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
2594
2595   leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, a);
2596   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
2597   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
2598
2599   lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2600
2601   return lbi0 == ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0), a);
2602 }
2603
2604 static clib_error_t *
2605 test_lookup_command_fn (vlib_main_t * vm,
2606                         unformat_input_t * input, vlib_cli_command_t * cmd)
2607 {
2608   ip4_fib_t *fib;
2609   u32 table_id = 0;
2610   f64 count = 1;
2611   u32 n;
2612   int i;
2613   ip4_address_t ip4_base_address;
2614   u64 errors = 0;
2615
2616   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2617     {
2618       if (unformat (input, "table %d", &table_id))
2619         {
2620           /* Make sure the entry exists. */
2621           fib = ip4_fib_get (table_id);
2622           if ((fib) && (fib->index != table_id))
2623             return clib_error_return (0, "<fib-index> %d does not exist",
2624                                       table_id);
2625         }
2626       else if (unformat (input, "count %f", &count))
2627         ;
2628
2629       else if (unformat (input, "%U",
2630                          unformat_ip4_address, &ip4_base_address))
2631         ;
2632       else
2633         return clib_error_return (0, "unknown input `%U'",
2634                                   format_unformat_error, input);
2635     }
2636
2637   n = count;
2638
2639   for (i = 0; i < n; i++)
2640     {
2641       if (!ip4_lookup_validate (&ip4_base_address, table_id))
2642         errors++;
2643
2644       ip4_base_address.as_u32 =
2645         clib_host_to_net_u32 (1 +
2646                               clib_net_to_host_u32 (ip4_base_address.as_u32));
2647     }
2648
2649   if (errors)
2650     vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
2651   else
2652     vlib_cli_output (vm, "No errors in %d lookups\n", n);
2653
2654   return 0;
2655 }
2656
2657 /*?
2658  * Perform a lookup of an IPv4 Address (or range of addresses) in the
2659  * given FIB table to determine if there is a conflict with the
2660  * adjacency table. The fib-id can be determined by using the
2661  * '<em>show ip fib</em>' command. If fib-id is not entered, default value
2662  * of 0 is used.
2663  *
2664  * @todo This command uses fib-id, other commands use table-id (not
2665  * just a name, they are different indexes). Would like to change this
2666  * to table-id for consistency.
2667  *
2668  * @cliexpar
2669  * Example of how to run the test lookup command:
2670  * @cliexstart{test lookup 172.16.1.1 table 1 count 2}
2671  * No errors in 2 lookups
2672  * @cliexend
2673 ?*/
2674 /* *INDENT-OFF* */
2675 VLIB_CLI_COMMAND (lookup_test_command, static) =
2676 {
2677   .path = "test lookup",
2678   .short_help = "test lookup <ipv4-addr> [table <fib-id>] [count <nn>]",
2679   .function = test_lookup_command_fn,
2680 };
2681 /* *INDENT-ON* */
2682
2683 #ifndef CLIB_MARCH_VARIANT
2684 int
2685 vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
2686 {
2687   u32 fib_index;
2688
2689   fib_index = fib_table_find (FIB_PROTOCOL_IP4, table_id);
2690
2691   if (~0 == fib_index)
2692     return VNET_API_ERROR_NO_SUCH_FIB;
2693
2694   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP4,
2695                                   flow_hash_config);
2696
2697   return 0;
2698 }
2699 #endif
2700
2701 static clib_error_t *
2702 set_ip_flow_hash_command_fn (vlib_main_t * vm,
2703                              unformat_input_t * input,
2704                              vlib_cli_command_t * cmd)
2705 {
2706   int matched = 0;
2707   u32 table_id = 0;
2708   u32 flow_hash_config = 0;
2709   int rv;
2710
2711   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2712     {
2713       if (unformat (input, "table %d", &table_id))
2714         matched = 1;
2715 #define _(a,v) \
2716     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2717       foreach_flow_hash_bit
2718 #undef _
2719         else
2720         break;
2721     }
2722
2723   if (matched == 0)
2724     return clib_error_return (0, "unknown input `%U'",
2725                               format_unformat_error, input);
2726
2727   rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
2728   switch (rv)
2729     {
2730     case 0:
2731       break;
2732
2733     case VNET_API_ERROR_NO_SUCH_FIB:
2734       return clib_error_return (0, "no such FIB table %d", table_id);
2735
2736     default:
2737       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2738       break;
2739     }
2740
2741   return 0;
2742 }
2743
2744 /*?
2745  * Configure the set of IPv4 fields used by the flow hash.
2746  *
2747  * @cliexpar
2748  * Example of how to set the flow hash on a given table:
2749  * @cliexcmd{set ip flow-hash table 7 dst sport dport proto}
2750  * Example of display the configured flow hash:
2751  * @cliexstart{show ip fib}
2752  * ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2753  * 0.0.0.0/0
2754  *   unicast-ip4-chain
2755  *   [@0]: dpo-load-balance: [index:0 buckets:1 uRPF:0 to:[0:0]]
2756  *     [0] [@0]: dpo-drop ip6
2757  * 0.0.0.0/32
2758  *   unicast-ip4-chain
2759  *   [@0]: dpo-load-balance: [index:1 buckets:1 uRPF:1 to:[0:0]]
2760  *     [0] [@0]: dpo-drop ip6
2761  * 224.0.0.0/8
2762  *   unicast-ip4-chain
2763  *   [@0]: dpo-load-balance: [index:3 buckets:1 uRPF:3 to:[0:0]]
2764  *     [0] [@0]: dpo-drop ip6
2765  * 6.0.1.2/32
2766  *   unicast-ip4-chain
2767  *   [@0]: dpo-load-balance: [index:30 buckets:1 uRPF:29 to:[0:0]]
2768  *     [0] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
2769  * 7.0.0.1/32
2770  *   unicast-ip4-chain
2771  *   [@0]: dpo-load-balance: [index:31 buckets:4 uRPF:30 to:[0:0]]
2772  *     [0] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
2773  *     [1] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
2774  *     [2] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
2775  *     [3] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
2776  * 240.0.0.0/8
2777  *   unicast-ip4-chain
2778  *   [@0]: dpo-load-balance: [index:2 buckets:1 uRPF:2 to:[0:0]]
2779  *     [0] [@0]: dpo-drop ip6
2780  * 255.255.255.255/32
2781  *   unicast-ip4-chain
2782  *   [@0]: dpo-load-balance: [index:4 buckets:1 uRPF:4 to:[0:0]]
2783  *     [0] [@0]: dpo-drop ip6
2784  * ipv4-VRF:7, fib_index 1, flow hash: dst sport dport proto
2785  * 0.0.0.0/0
2786  *   unicast-ip4-chain
2787  *   [@0]: dpo-load-balance: [index:12 buckets:1 uRPF:11 to:[0:0]]
2788  *     [0] [@0]: dpo-drop ip6
2789  * 0.0.0.0/32
2790  *   unicast-ip4-chain
2791  *   [@0]: dpo-load-balance: [index:13 buckets:1 uRPF:12 to:[0:0]]
2792  *     [0] [@0]: dpo-drop ip6
2793  * 172.16.1.0/24
2794  *   unicast-ip4-chain
2795  *   [@0]: dpo-load-balance: [index:17 buckets:1 uRPF:16 to:[0:0]]
2796  *     [0] [@4]: ipv4-glean: af_packet0
2797  * 172.16.1.1/32
2798  *   unicast-ip4-chain
2799  *   [@0]: dpo-load-balance: [index:18 buckets:1 uRPF:17 to:[1:84]]
2800  *     [0] [@2]: dpo-receive: 172.16.1.1 on af_packet0
2801  * 172.16.1.2/32
2802  *   unicast-ip4-chain
2803  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
2804  *     [0] [@5]: ipv4 via 172.16.1.2 af_packet0: IP4: 02:fe:9e:70:7a:2b -> 26:a5:f6:9c:3a:36
2805  * 172.16.2.0/24
2806  *   unicast-ip4-chain
2807  *   [@0]: dpo-load-balance: [index:19 buckets:1 uRPF:18 to:[0:0]]
2808  *     [0] [@4]: ipv4-glean: af_packet1
2809  * 172.16.2.1/32
2810  *   unicast-ip4-chain
2811  *   [@0]: dpo-load-balance: [index:20 buckets:1 uRPF:19 to:[0:0]]
2812  *     [0] [@2]: dpo-receive: 172.16.2.1 on af_packet1
2813  * 224.0.0.0/8
2814  *   unicast-ip4-chain
2815  *   [@0]: dpo-load-balance: [index:15 buckets:1 uRPF:14 to:[0:0]]
2816  *     [0] [@0]: dpo-drop ip6
2817  * 240.0.0.0/8
2818  *   unicast-ip4-chain
2819  *   [@0]: dpo-load-balance: [index:14 buckets:1 uRPF:13 to:[0:0]]
2820  *     [0] [@0]: dpo-drop ip6
2821  * 255.255.255.255/32
2822  *   unicast-ip4-chain
2823  *   [@0]: dpo-load-balance: [index:16 buckets:1 uRPF:15 to:[0:0]]
2824  *     [0] [@0]: dpo-drop ip6
2825  * @cliexend
2826 ?*/
2827 /* *INDENT-OFF* */
2828 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) =
2829 {
2830   .path = "set ip flow-hash",
2831   .short_help =
2832   "set ip flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
2833   .function = set_ip_flow_hash_command_fn,
2834 };
2835 /* *INDENT-ON* */
2836
2837 #ifndef CLIB_MARCH_VARIANT
2838 int
2839 vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
2840                              u32 table_index)
2841 {
2842   vnet_main_t *vnm = vnet_get_main ();
2843   vnet_interface_main_t *im = &vnm->interface_main;
2844   ip4_main_t *ipm = &ip4_main;
2845   ip_lookup_main_t *lm = &ipm->lookup_main;
2846   vnet_classify_main_t *cm = &vnet_classify_main;
2847   ip4_address_t *if_addr;
2848
2849   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
2850     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
2851
2852   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
2853     return VNET_API_ERROR_NO_SUCH_ENTRY;
2854
2855   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
2856   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
2857
2858   if_addr = ip4_interface_first_address (ipm, sw_if_index, NULL);
2859
2860   if (NULL != if_addr)
2861     {
2862       fib_prefix_t pfx = {
2863         .fp_len = 32,
2864         .fp_proto = FIB_PROTOCOL_IP4,
2865         .fp_addr.ip4 = *if_addr,
2866       };
2867       u32 fib_index;
2868
2869       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2870                                                        sw_if_index);
2871
2872
2873       if (table_index != (u32) ~ 0)
2874         {
2875           dpo_id_t dpo = DPO_INVALID;
2876
2877           dpo_set (&dpo,
2878                    DPO_CLASSIFY,
2879                    DPO_PROTO_IP4,
2880                    classify_dpo_create (DPO_PROTO_IP4, table_index));
2881
2882           fib_table_entry_special_dpo_add (fib_index,
2883                                            &pfx,
2884                                            FIB_SOURCE_CLASSIFY,
2885                                            FIB_ENTRY_FLAG_NONE, &dpo);
2886           dpo_reset (&dpo);
2887         }
2888       else
2889         {
2890           fib_table_entry_special_remove (fib_index,
2891                                           &pfx, FIB_SOURCE_CLASSIFY);
2892         }
2893     }
2894
2895   return 0;
2896 }
2897 #endif
2898
2899 static clib_error_t *
2900 set_ip_classify_command_fn (vlib_main_t * vm,
2901                             unformat_input_t * input,
2902                             vlib_cli_command_t * cmd)
2903 {
2904   u32 table_index = ~0;
2905   int table_index_set = 0;
2906   u32 sw_if_index = ~0;
2907   int rv;
2908
2909   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2910     {
2911       if (unformat (input, "table-index %d", &table_index))
2912         table_index_set = 1;
2913       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
2914                          vnet_get_main (), &sw_if_index))
2915         ;
2916       else
2917         break;
2918     }
2919
2920   if (table_index_set == 0)
2921     return clib_error_return (0, "classify table-index must be specified");
2922
2923   if (sw_if_index == ~0)
2924     return clib_error_return (0, "interface / subif must be specified");
2925
2926   rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
2927
2928   switch (rv)
2929     {
2930     case 0:
2931       break;
2932
2933     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
2934       return clib_error_return (0, "No such interface");
2935
2936     case VNET_API_ERROR_NO_SUCH_ENTRY:
2937       return clib_error_return (0, "No such classifier table");
2938     }
2939   return 0;
2940 }
2941
2942 /*?
2943  * Assign a classification table to an interface. The classification
2944  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
2945  * commands. Once the table is create, use this command to filter packets
2946  * on an interface.
2947  *
2948  * @cliexpar
2949  * Example of how to assign a classification table to an interface:
2950  * @cliexcmd{set ip classify intfc GigabitEthernet2/0/0 table-index 1}
2951 ?*/
2952 /* *INDENT-OFF* */
2953 VLIB_CLI_COMMAND (set_ip_classify_command, static) =
2954 {
2955     .path = "set ip classify",
2956     .short_help =
2957     "set ip classify intfc <interface> table-index <classify-idx>",
2958     .function = set_ip_classify_command_fn,
2959 };
2960 /* *INDENT-ON* */
2961
2962 static clib_error_t *
2963 ip4_config (vlib_main_t * vm, unformat_input_t * input)
2964 {
2965   ip4_main_t *im = &ip4_main;
2966   uword heapsize = 0;
2967
2968   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2969     {
2970       if (unformat (input, "heap-size %U", unformat_memory_size, &heapsize))
2971         ;
2972       else
2973         return clib_error_return (0,
2974                                   "invalid heap-size parameter `%U'",
2975                                   format_unformat_error, input);
2976     }
2977
2978   im->mtrie_heap_size = heapsize;
2979
2980   return 0;
2981 }
2982
2983 VLIB_EARLY_CONFIG_FUNCTION (ip4_config, "ip");
2984
2985 /*
2986  * fd.io coding-style-patch-verification: ON
2987  *
2988  * Local Variables:
2989  * eval: (c-set-style "gnu")
2990  * End:
2991  */