vnet/vnet/ip/ip4_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip4_forward.c: IP v4 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  43 #include <vnet/ethernet/arp_packet.h>   /* for ethernet_arp_header_t */
  44 #include <vnet/ppp/ppp.h>
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vnet/api_errno.h>     /* for API error numbers */
  47
  48 /* This is really, really simple but stupid fib. */
  49 u32
  50 ip4_fib_lookup_with_table (ip4_main_t * im, u32 fib_index,
  51                            ip4_address_t * dst,
  52                            u32 disable_default_route)
  53 {
  54   ip_lookup_main_t * lm = &im->lookup_main;
  55   ip4_fib_t * fib = vec_elt_at_index (im->fibs, fib_index);
  56   uword * p, * hash, key;
  57   i32 i, i_min, dst_address, ai;
  58
  59   i_min = disable_default_route ? 1 : 0;
  60   dst_address = clib_mem_unaligned (&dst->data_u32, u32);
  61   for (i = ARRAY_LEN (fib->adj_index_by_dst_address) - 1; i >= i_min; i--)
  62     {
  63       hash = fib->adj_index_by_dst_address[i];
  64       if (! hash)
  65         continue;
  66
  67       key = dst_address & im->fib_masks[i];
  68       if ((p = hash_get (hash, key)) != 0)
  69         {
  70           ai = p[0];
  71           goto done;
  72         }
  73     }
  74
  75   /* Nothing matches in table. */
  76   ai = lm->miss_adj_index;
  77
  78  done:
  79   return ai;
  80 }
  81
  82 static ip4_fib_t *
  83 create_fib_with_table_id (ip4_main_t * im, u32 table_id)
  84 {
  85   ip4_fib_t * fib;
  86   hash_set (im->fib_index_by_table_id, table_id, vec_len (im->fibs));
  87   vec_add2 (im->fibs, fib, 1);
  88   fib->table_id = table_id;
  89   fib->index = fib - im->fibs;
  90   fib->flow_hash_config = IP_FLOW_HASH_DEFAULT;
  91   fib->fwd_classify_table_index = ~0;
  92   fib->rev_classify_table_index = ~0;
  93   ip4_mtrie_init (&fib->mtrie);
  94   return fib;
  95 }
  96
  97 ip4_fib_t *
  98 find_ip4_fib_by_table_index_or_id (ip4_main_t * im,
  99                                    u32 table_index_or_id, u32 flags)
 100 {
 101   uword * p, fib_index;
 102
 103   fib_index = table_index_or_id;
 104   if (! (flags & IP4_ROUTE_FLAG_FIB_INDEX))
 105     {
 106       if (table_index_or_id == ~0) {
 107         table_index_or_id = 0;
 108         while ((p = hash_get (im->fib_index_by_table_id, table_index_or_id))) {
 109           table_index_or_id++;
 110         }
 111         return create_fib_with_table_id (im, table_index_or_id);
 112       }
 113
 114       p = hash_get (im->fib_index_by_table_id, table_index_or_id);
 115       if (! p)
 116         return create_fib_with_table_id (im, table_index_or_id);
 117       fib_index = p[0];
 118     }
 119   return vec_elt_at_index (im->fibs, fib_index);
 120 }
 121
 122 static void
 123 ip4_fib_init_adj_index_by_dst_address (ip_lookup_main_t * lm,
 124                                        ip4_fib_t * fib,
 125                                        u32 address_length)
 126 {
 127   hash_t * h;
 128   uword max_index;
 129
 130   ASSERT (lm->fib_result_n_bytes >= sizeof (uword));
 131   lm->fib_result_n_words = round_pow2 (lm->fib_result_n_bytes, sizeof (uword)) / sizeof (uword);
 132
 133   fib->adj_index_by_dst_address[address_length] =
 134     hash_create (32 /* elts */, lm->fib_result_n_words * sizeof (uword));
 135
 136   hash_set_flags (fib->adj_index_by_dst_address[address_length],
 137                   HASH_FLAG_NO_AUTO_SHRINK);
 138
 139   h = hash_header (fib->adj_index_by_dst_address[address_length]);
 140   max_index = (hash_value_bytes (h) / sizeof (fib->new_hash_values[0])) - 1;
 141
 142   /* Initialize new/old hash value vectors. */
 143   vec_validate_init_empty (fib->new_hash_values, max_index, ~0);
 144   vec_validate_init_empty (fib->old_hash_values, max_index, ~0);
 145 }
 146
 147 static void
 148 ip4_fib_set_adj_index (ip4_main_t * im,
 149                        ip4_fib_t * fib,
 150                        u32 flags,
 151                        u32 dst_address_u32,
 152                        u32 dst_address_length,
 153                        u32 adj_index)
 154 {
 155   ip_lookup_main_t * lm = &im->lookup_main;
 156   uword * hash;
 157
 158   if (vec_bytes(fib->old_hash_values))
 159     memset (fib->old_hash_values, ~0, vec_bytes (fib->old_hash_values));
 160   if (vec_bytes(fib->new_hash_values))
 161     memset (fib->new_hash_values, ~0, vec_bytes (fib->new_hash_values));
 162   fib->new_hash_values[0] = adj_index;
 163
 164   /* Make sure adj index is valid. */
 165   if (CLIB_DEBUG > 0)
 166     (void) ip_get_adjacency (lm, adj_index);
 167
 168   hash = fib->adj_index_by_dst_address[dst_address_length];
 169
 170   hash = _hash_set3 (hash, dst_address_u32,
 171                      fib->new_hash_values,
 172                      fib->old_hash_values);
 173
 174   fib->adj_index_by_dst_address[dst_address_length] = hash;
 175
 176   if (vec_len (im->add_del_route_callbacks) > 0)
 177     {
 178       ip4_add_del_route_callback_t * cb;
 179       ip4_address_t d;
 180       uword * p;
 181
 182       d.data_u32 = dst_address_u32;
 183       vec_foreach (cb, im->add_del_route_callbacks)
 184         if ((flags & cb->required_flags) == cb->required_flags)
 185           cb->function (im, cb->function_opaque,
 186                         fib, flags,
 187                         &d, dst_address_length,
 188                         fib->old_hash_values,
 189                         fib->new_hash_values);
 190
 191       p = hash_get (hash, dst_address_u32);
 192       clib_memcpy (p, fib->new_hash_values, vec_bytes (fib->new_hash_values));
 193     }
 194 }
 195
 196 void ip4_add_del_route (ip4_main_t * im, ip4_add_del_route_args_t * a)
 197 {
 198   ip_lookup_main_t * lm = &im->lookup_main;
 199   ip4_fib_t * fib;
 200   u32 dst_address, dst_address_length, adj_index, old_adj_index;
 201   uword * hash, is_del;
 202   ip4_add_del_route_callback_t * cb;
 203
 204   /* Either create new adjacency or use given one depending on arguments. */
 205   if (a->n_add_adj > 0)
 206     {
 207       ip_add_adjacency (lm, a->add_adj, a->n_add_adj, &adj_index);
 208       ip_call_add_del_adjacency_callbacks (lm, adj_index, /* is_del */ 0);
 209     }
 210   else
 211     adj_index = a->adj_index;
 212
 213   dst_address = a->dst_address.data_u32;
 214   dst_address_length = a->dst_address_length;
 215   fib = find_ip4_fib_by_table_index_or_id (im, a->table_index_or_table_id, a->flags);
 216
 217   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 218   dst_address &= im->fib_masks[dst_address_length];
 219
 220   if (! fib->adj_index_by_dst_address[dst_address_length])
 221     ip4_fib_init_adj_index_by_dst_address (lm, fib, dst_address_length);
 222
 223   hash = fib->adj_index_by_dst_address[dst_address_length];
 224
 225   is_del = (a->flags & IP4_ROUTE_FLAG_DEL) != 0;
 226
 227   if (is_del)
 228     {
 229       fib->old_hash_values[0] = ~0;
 230       hash = _hash_unset (hash, dst_address, fib->old_hash_values);
 231       fib->adj_index_by_dst_address[dst_address_length] = hash;
 232
 233       if (vec_len (im->add_del_route_callbacks) > 0
 234           && fib->old_hash_values[0] != ~0) /* make sure destination was found in hash */
 235         {
 236           fib->new_hash_values[0] = ~0;
 237           vec_foreach (cb, im->add_del_route_callbacks)
 238             if ((a->flags & cb->required_flags) == cb->required_flags)
 239               cb->function (im, cb->function_opaque,
 240                             fib, a->flags,
 241                             &a->dst_address, dst_address_length,
 242                             fib->old_hash_values,
 243                             fib->new_hash_values);
 244         }
 245     }
 246   else
 247     ip4_fib_set_adj_index (im, fib, a->flags, dst_address, dst_address_length,
 248                            adj_index);
 249
 250   old_adj_index = fib->old_hash_values[0];
 251
 252   /* Avoid spurious reference count increments */
 253   if (old_adj_index == adj_index
 254       && adj_index != ~0
 255       && !(a->flags & IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY))
 256     {
 257       ip_adjacency_t * adj = ip_get_adjacency (lm, adj_index);
 258       if (adj->share_count > 0)
 259         adj->share_count --;
 260     }
 261
 262   ip4_fib_mtrie_add_del_route (fib, a->dst_address, dst_address_length,
 263                                is_del ? old_adj_index : adj_index,
 264                                is_del);
 265
 266   /* Delete old adjacency index if present and changed. */
 267   if (! (a->flags & IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY)
 268       && old_adj_index != ~0
 269       && old_adj_index != adj_index)
 270     ip_del_adjacency (lm, old_adj_index);
 271 }
 272
 273 void
 274 ip4_add_del_route_next_hop (ip4_main_t * im,
 275                             u32 flags,
 276                             ip4_address_t * dst_address,
 277                             u32 dst_address_length,
 278                             ip4_address_t * next_hop,
 279                             u32 next_hop_sw_if_index,
 280                             u32 next_hop_weight, u32 adj_index,
 281                             u32 explicit_fib_index)
 282 {
 283   vnet_main_t * vnm = vnet_get_main();
 284   ip_lookup_main_t * lm = &im->lookup_main;
 285   u32 fib_index;
 286   ip4_fib_t * fib;
 287   u32 dst_address_u32, old_mp_adj_index, new_mp_adj_index;
 288   u32 dst_adj_index, nh_adj_index;
 289   uword * dst_hash, * dst_result;
 290   uword * nh_hash, * nh_result;
 291   ip_adjacency_t * dst_adj;
 292   ip_multipath_adjacency_t * old_mp, * new_mp;
 293   int is_del = (flags & IP4_ROUTE_FLAG_DEL) != 0;
 294   int is_interface_next_hop;
 295   clib_error_t * error = 0;
 296
 297   if (explicit_fib_index == (u32)~0)
 298       fib_index = vec_elt (im->fib_index_by_sw_if_index, next_hop_sw_if_index);
 299   else
 300       fib_index = explicit_fib_index;
 301
 302   fib = vec_elt_at_index (im->fibs, fib_index);
 303
 304   /* Lookup next hop to be added or deleted. */
 305   is_interface_next_hop = next_hop->data_u32 == 0;
 306   if (adj_index == (u32)~0)
 307     {
 308       if (is_interface_next_hop)
 309         {
 310           nh_result = hash_get (im->interface_route_adj_index_by_sw_if_index, next_hop_sw_if_index);
 311           if (nh_result)
 312             nh_adj_index = *nh_result;
 313           else
 314             {
 315               ip_adjacency_t * adj;
 316               adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 317                                       &nh_adj_index);
 318               ip4_adjacency_set_interface_route (vnm, adj, next_hop_sw_if_index, /* if_address_index */ ~0);
 319               ip_call_add_del_adjacency_callbacks (lm, nh_adj_index, /* is_del */ 0);
 320               hash_set (im->interface_route_adj_index_by_sw_if_index, next_hop_sw_if_index, nh_adj_index);
 321             }
 322         }
 323       else
 324         {
 325           nh_hash = fib->adj_index_by_dst_address[32];
 326           nh_result = hash_get (nh_hash, next_hop->data_u32);
 327
 328           /* Next hop must be known. */
 329           if (! nh_result)
 330             {
 331               ip_adjacency_t * adj;
 332
 333               nh_adj_index = ip4_fib_lookup_with_table (im, fib_index,
 334                                                         next_hop, 0);
 335               adj = ip_get_adjacency (lm, nh_adj_index);
 336               /* if ARP interface adjacencty is present, we need to
 337                  install ARP adjaceny for specific next hop */
 338               if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP &&
 339                   adj->arp.next_hop.ip4.as_u32 == 0)
 340                 {
 341                   nh_adj_index = vnet_arp_glean_add(fib_index, next_hop);
 342                 }
 343               else
 344                 {
 345                   /* Next hop is not known, so create indirect adj */
 346                   ip_adjacency_t add_adj;
 347                   memset (&add_adj, 0, sizeof(add_adj));
 348                   add_adj.n_adj = 1;
 349                   add_adj.lookup_next_index = IP_LOOKUP_NEXT_INDIRECT;
 350                   add_adj.indirect.next_hop.ip4.as_u32 = next_hop->as_u32;
 351                   add_adj.explicit_fib_index = explicit_fib_index;
 352                   ip_add_adjacency (lm, &add_adj, 1, &nh_adj_index);
 353                 }
 354             }
 355           else
 356             nh_adj_index = *nh_result;
 357         }
 358     }
 359   else
 360     {
 361       nh_adj_index = adj_index;
 362     }
 363   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 364   dst_address_u32 = dst_address->data_u32 & im->fib_masks[dst_address_length];
 365
 366   dst_hash = fib->adj_index_by_dst_address[dst_address_length];
 367   dst_result = hash_get (dst_hash, dst_address_u32);
 368   if (dst_result)
 369     {
 370       dst_adj_index = dst_result[0];
 371       dst_adj = ip_get_adjacency (lm, dst_adj_index);
 372     }
 373   else
 374     {
 375       /* For deletes destination must be known. */
 376       if (is_del)
 377         {
 378           vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
 379           error = clib_error_return (0, "unknown destination %U/%d",
 380                                      format_ip4_address, dst_address,
 381                                      dst_address_length);
 382           goto done;
 383         }
 384
 385       dst_adj_index = ~0;
 386       dst_adj = 0;
 387     }
 388
 389   /* Ignore adds of X/32 with next hop of X. */
 390   if (! is_del
 391       && dst_address_length == 32
 392       && dst_address->data_u32 == next_hop->data_u32
 393       && adj_index != (u32)~0)
 394     {
 395       vnm->api_errno = VNET_API_ERROR_PREFIX_MATCHES_NEXT_HOP;
 396       error = clib_error_return (0, "prefix matches next hop %U/%d",
 397                                  format_ip4_address, dst_address,
 398                                  dst_address_length);
 399       goto done;
 400     }
 401
 402   /* Destination is not known and default weight is set so add route
 403      to existing non-multipath adjacency */
 404   if (dst_adj_index == ~0 && next_hop_weight == 1 && next_hop_sw_if_index == ~0)
 405     {
 406       /* create new adjacency */
 407       ip4_add_del_route_args_t a;
 408       a.table_index_or_table_id = fib_index;
 409       a.flags = ((is_del ? IP4_ROUTE_FLAG_DEL : IP4_ROUTE_FLAG_ADD)
 410                  | IP4_ROUTE_FLAG_FIB_INDEX
 411                  | IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 412                  | (flags & (IP4_ROUTE_FLAG_NO_REDISTRIBUTE
 413                              | IP4_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
 414       a.dst_address = dst_address[0];
 415       a.dst_address_length = dst_address_length;
 416       a.adj_index = nh_adj_index;
 417       a.add_adj = 0;
 418       a.n_add_adj = 0;
 419
 420       ip4_add_del_route (im, &a);
 421
 422       goto done;
 423     }
 424
 425   old_mp_adj_index = dst_adj ? dst_adj->heap_handle : ~0;
 426
 427   if (! ip_multipath_adjacency_add_del_next_hop
 428       (lm, is_del,
 429        old_mp_adj_index,
 430        nh_adj_index,
 431        next_hop_weight,
 432        &new_mp_adj_index))
 433     {
 434       vnm->api_errno = VNET_API_ERROR_NEXT_HOP_NOT_FOUND_MP;
 435       error = clib_error_return (0, "requested deleting next-hop %U not found in multi-path",
 436                                  format_ip4_address, next_hop);
 437       goto done;
 438     }
 439
 440   old_mp = new_mp = 0;
 441   if (old_mp_adj_index != ~0)
 442     old_mp = vec_elt_at_index (lm->multipath_adjacencies, old_mp_adj_index);
 443   if (new_mp_adj_index != ~0)
 444     new_mp = vec_elt_at_index (lm->multipath_adjacencies, new_mp_adj_index);
 445
 446   if (old_mp != new_mp)
 447     {
 448       ip4_add_del_route_args_t a;
 449       a.table_index_or_table_id = fib_index;
 450       a.flags = ((is_del && ! new_mp ? IP4_ROUTE_FLAG_DEL : IP4_ROUTE_FLAG_ADD)
 451                  | IP4_ROUTE_FLAG_FIB_INDEX
 452                  | IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 453                  | (flags & (IP4_ROUTE_FLAG_NO_REDISTRIBUTE | IP4_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
 454       a.dst_address = dst_address[0];
 455       a.dst_address_length = dst_address_length;
 456       a.adj_index = new_mp ? new_mp->adj_index : dst_adj_index;
 457       a.add_adj = 0;
 458       a.n_add_adj = 0;
 459
 460       ip4_add_del_route (im, &a);
 461     }
 462
 463  done:
 464   if (error)
 465     clib_error_report (error);
 466 }
 467
 468 void *
 469 ip4_get_route (ip4_main_t * im,
 470                u32 table_index_or_table_id,
 471                u32 flags,
 472                u8 * address,
 473                u32 address_length)
 474 {
 475   ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 476   u32 dst_address = * (u32 *) address;
 477   uword * hash, * p;
 478
 479   ASSERT (address_length < ARRAY_LEN (im->fib_masks));
 480   dst_address &= im->fib_masks[address_length];
 481
 482   hash = fib->adj_index_by_dst_address[address_length];
 483   p = hash_get (hash, dst_address);
 484   return (void *) p;
 485 }
 486
 487 void
 488 ip4_foreach_matching_route (ip4_main_t * im,
 489                             u32 table_index_or_table_id,
 490                             u32 flags,
 491                             ip4_address_t * address,
 492                             u32 address_length,
 493                             ip4_address_t ** results,
 494                             u8 ** result_lengths)
 495 {
 496   ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 497   u32 dst_address = address->data_u32;
 498   u32 this_length = address_length;
 499
 500   if (*results)
 501     _vec_len (*results) = 0;
 502   if (*result_lengths)
 503     _vec_len (*result_lengths) = 0;
 504
 505   while (this_length <= 32 && vec_len (results) == 0)
 506     {
 507       uword k, v;
 508       hash_foreach (k, v, fib->adj_index_by_dst_address[this_length], ({
 509         if (0 == ((k ^ dst_address) & im->fib_masks[address_length]))
 510           {
 511             ip4_address_t a;
 512             a.data_u32 = k;
 513             vec_add1 (*results, a);
 514             vec_add1 (*result_lengths, this_length);
 515           }
 516       }));
 517
 518       this_length++;
 519     }
 520 }
 521
 522 void ip4_maybe_remap_adjacencies (ip4_main_t * im,
 523                                   u32 table_index_or_table_id,
 524                                   u32 flags)
 525 {
 526   ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 527   ip_lookup_main_t * lm = &im->lookup_main;
 528   u32 i, l;
 529   ip4_address_t a;
 530   ip4_add_del_route_callback_t * cb;
 531   static ip4_address_t * to_delete;
 532
 533   if (lm->n_adjacency_remaps == 0)
 534     return;
 535
 536   for (l = 0; l <= 32; l++)
 537     {
 538       hash_pair_t * p;
 539       uword * hash = fib->adj_index_by_dst_address[l];
 540
 541       if (hash_elts (hash) == 0)
 542         continue;
 543
 544       if (to_delete)
 545         _vec_len (to_delete) = 0;
 546
 547       hash_foreach_pair (p, hash, ({
 548         u32 adj_index = p->value[0];
 549         u32 m = vec_elt (lm->adjacency_remap_table, adj_index);
 550
 551         if (m)
 552           {
 553             /* Record destination address from hash key. */
 554             a.data_u32 = p->key;
 555
 556             /* New adjacency points to nothing: so delete prefix. */
 557             if (m == ~0)
 558               vec_add1 (to_delete, a);
 559             else
 560               {
 561                 /* Remap to new adjacency. */
 562                 clib_memcpy (fib->old_hash_values, p->value, vec_bytes (fib->old_hash_values));
 563
 564                 /* Set new adjacency value. */
 565                 fib->new_hash_values[0] = p->value[0] = m - 1;
 566
 567                 vec_foreach (cb, im->add_del_route_callbacks)
 568                   if ((flags & cb->required_flags) == cb->required_flags)
 569                     cb->function (im, cb->function_opaque,
 570                                   fib, flags | IP4_ROUTE_FLAG_ADD,
 571                                   &a, l,
 572                                   fib->old_hash_values,
 573                                   fib->new_hash_values);
 574               }
 575           }
 576       }));
 577
 578       fib->new_hash_values[0] = ~0;
 579       for (i = 0; i < vec_len (to_delete); i++)
 580         {
 581           hash = _hash_unset (hash, to_delete[i].data_u32, fib->old_hash_values);
 582           vec_foreach (cb, im->add_del_route_callbacks)
 583             if ((flags & cb->required_flags) == cb->required_flags)
 584               cb->function (im, cb->function_opaque,
 585                             fib, flags | IP4_ROUTE_FLAG_DEL,
 586                             &a, l,
 587                             fib->old_hash_values,
 588                             fib->new_hash_values);
 589         }
 590     }
 591
 592   /* Also remap adjacencies in mtrie. */
 593   ip4_mtrie_maybe_remap_adjacencies (lm, &fib->mtrie);
 594
 595   /* Reset mapping table. */
 596   vec_zero (lm->adjacency_remap_table);
 597
 598   /* All remaps have been performed. */
 599   lm->n_adjacency_remaps = 0;
 600 }
 601
 602 void ip4_delete_matching_routes (ip4_main_t * im,
 603                                  u32 table_index_or_table_id,
 604                                  u32 flags,
 605                                  ip4_address_t * address,
 606                                  u32 address_length)
 607 {
 608   static ip4_address_t * matching_addresses;
 609   static u8 * matching_address_lengths;
 610   u32 l, i;
 611   ip4_add_del_route_args_t a;
 612
 613   a.flags = IP4_ROUTE_FLAG_DEL | IP4_ROUTE_FLAG_NO_REDISTRIBUTE | flags;
 614   a.table_index_or_table_id = table_index_or_table_id;
 615   a.adj_index = ~0;
 616   a.add_adj = 0;
 617   a.n_add_adj = 0;
 618
 619   for (l = address_length + 1; l <= 32; l++)
 620     {
 621       ip4_foreach_matching_route (im, table_index_or_table_id, flags,
 622                                   address,
 623                                   l,
 624                                   &matching_addresses,
 625                                   &matching_address_lengths);
 626       for (i = 0; i < vec_len (matching_addresses); i++)
 627         {
 628           a.dst_address = matching_addresses[i];
 629           a.dst_address_length = matching_address_lengths[i];
 630           ip4_add_del_route (im, &a);
 631         }
 632     }
 633
 634   ip4_maybe_remap_adjacencies (im, table_index_or_table_id, flags);
 635 }
 636
 637 void
 638 ip4_forward_next_trace (vlib_main_t * vm,
 639                         vlib_node_runtime_t * node,
 640                         vlib_frame_t * frame,
 641                         vlib_rx_or_tx_t which_adj_index);
 642
 643 always_inline uword
 644 ip4_lookup_inline (vlib_main_t * vm,
 645                    vlib_node_runtime_t * node,
 646                    vlib_frame_t * frame,
 647                    int lookup_for_responses_to_locally_received_packets,
 648                    int is_indirect)
 649 {
 650   ip4_main_t * im = &ip4_main;
 651   ip_lookup_main_t * lm = &im->lookup_main;
 652   vlib_combined_counter_main_t * cm = &im->lookup_main.adjacency_counters;
 653   u32 n_left_from, n_left_to_next, * from, * to_next;
 654   ip_lookup_next_t next;
 655   u32 cpu_index = os_get_cpu_number();
 656
 657   from = vlib_frame_vector_args (frame);
 658   n_left_from = frame->n_vectors;
 659   next = node->cached_next_index;
 660
 661   while (n_left_from > 0)
 662     {
 663       vlib_get_next_frame (vm, node, next,
 664                            to_next, n_left_to_next);
 665
 666       while (n_left_from >= 4 && n_left_to_next >= 2)
 667         {
 668           vlib_buffer_t * p0, * p1;
 669           ip4_header_t * ip0, * ip1;
 670           __attribute__((unused)) tcp_header_t * tcp0, * tcp1;
 671           ip_lookup_next_t next0, next1;
 672           ip_adjacency_t * adj0, * adj1;
 673           ip4_fib_mtrie_t * mtrie0, * mtrie1;
 674           ip4_fib_mtrie_leaf_t leaf0, leaf1;
 675           ip4_address_t * dst_addr0, *dst_addr1;
 676           __attribute__((unused)) u32 pi0, fib_index0, adj_index0, is_tcp_udp0;
 677           __attribute__((unused)) u32 pi1, fib_index1, adj_index1, is_tcp_udp1;
 678           u32 flow_hash_config0, flow_hash_config1;
 679           u32 hash_c0, hash_c1;
 680           u32 wrong_next;
 681
 682           /* Prefetch next iteration. */
 683           {
 684             vlib_buffer_t * p2, * p3;
 685
 686             p2 = vlib_get_buffer (vm, from[2]);
 687             p3 = vlib_get_buffer (vm, from[3]);
 688
 689             vlib_prefetch_buffer_header (p2, LOAD);
 690             vlib_prefetch_buffer_header (p3, LOAD);
 691
 692             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
 693             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
 694           }
 695
 696           pi0 = to_next[0] = from[0];
 697           pi1 = to_next[1] = from[1];
 698
 699           p0 = vlib_get_buffer (vm, pi0);
 700           p1 = vlib_get_buffer (vm, pi1);
 701
 702           ip0 = vlib_buffer_get_current (p0);
 703           ip1 = vlib_buffer_get_current (p1);
 704
 705           if (is_indirect)
 706             {
 707               ip_adjacency_t * iadj0, * iadj1;
 708               iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
 709               iadj1 = ip_get_adjacency (lm, vnet_buffer(p1)->ip.adj_index[VLIB_TX]);
 710               dst_addr0 = &iadj0->indirect.next_hop.ip4;
 711               dst_addr1 = &iadj1->indirect.next_hop.ip4;
 712             }
 713           else
 714             {
 715               dst_addr0 = &ip0->dst_address;
 716               dst_addr1 = &ip1->dst_address;
 717             }
 718
 719           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 720           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 721           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 722             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 723           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
 724             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
 725
 726
 727           if (! lookup_for_responses_to_locally_received_packets)
 728             {
 729               mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
 730               mtrie1 = &vec_elt_at_index (im->fibs, fib_index1)->mtrie;
 731
 732               leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
 733
 734               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 0);
 735               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 0);
 736             }
 737
 738           tcp0 = (void *) (ip0 + 1);
 739           tcp1 = (void *) (ip1 + 1);
 740
 741           is_tcp_udp0 = (ip0->protocol == IP_PROTOCOL_TCP
 742                          || ip0->protocol == IP_PROTOCOL_UDP);
 743           is_tcp_udp1 = (ip1->protocol == IP_PROTOCOL_TCP
 744                          || ip1->protocol == IP_PROTOCOL_UDP);
 745
 746           if (! lookup_for_responses_to_locally_received_packets)
 747             {
 748               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
 749               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 1);
 750             }
 751
 752           if (! lookup_for_responses_to_locally_received_packets)
 753             {
 754               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 755               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
 756             }
 757
 758           if (! lookup_for_responses_to_locally_received_packets)
 759             {
 760               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 761               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
 762             }
 763
 764           if (lookup_for_responses_to_locally_received_packets)
 765             {
 766               adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 767               adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
 768             }
 769           else
 770             {
 771               /* Handle default route. */
 772               leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 773               leaf1 = (leaf1 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie1->default_leaf : leaf1);
 774
 775               adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 776               adj_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 777             }
 778
 779           ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
 780                                                            dst_addr0,
 781                                                            /* no_default_route */ 0));
 782           ASSERT (adj_index1 == ip4_fib_lookup_with_table (im, fib_index1,
 783                                                            dst_addr1,
 784                                                            /* no_default_route */ 0));
 785           adj0 = ip_get_adjacency (lm, adj_index0);
 786           adj1 = ip_get_adjacency (lm, adj_index1);
 787
 788           next0 = adj0->lookup_next_index;
 789           next1 = adj1->lookup_next_index;
 790
 791           /* Use flow hash to compute multipath adjacency. */
 792           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 793           hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
 794           if (PREDICT_FALSE (adj0->n_adj > 1))
 795             {
 796               flow_hash_config0 =
 797                 vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
 798               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 799                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 800             }
 801           if (PREDICT_FALSE(adj1->n_adj > 1))
 802             {
 803               flow_hash_config1 =
 804                 vec_elt_at_index (im->fibs, fib_index1)->flow_hash_config;
 805               hash_c1 = vnet_buffer (p1)->ip.flow_hash =
 806                 ip4_compute_flow_hash (ip1, flow_hash_config1);
 807             }
 808
 809           ASSERT (adj0->n_adj > 0);
 810           ASSERT (adj1->n_adj > 0);
 811           ASSERT (is_pow2 (adj0->n_adj));
 812           ASSERT (is_pow2 (adj1->n_adj));
 813           adj_index0 += (hash_c0 & (adj0->n_adj - 1));
 814           adj_index1 += (hash_c1 & (adj1->n_adj - 1));
 815
 816           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 817           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
 818
 819           vlib_increment_combined_counter
 820               (cm, cpu_index, adj_index0, 1,
 821                vlib_buffer_length_in_chain (vm, p0)
 822                + sizeof(ethernet_header_t));
 823           vlib_increment_combined_counter
 824               (cm, cpu_index, adj_index1, 1,
 825                vlib_buffer_length_in_chain (vm, p1)
 826                + sizeof(ethernet_header_t));
 827
 828           from += 2;
 829           to_next += 2;
 830           n_left_to_next -= 2;
 831           n_left_from -= 2;
 832
 833           wrong_next = (next0 != next) + 2*(next1 != next);
 834           if (PREDICT_FALSE (wrong_next != 0))
 835             {
 836               switch (wrong_next)
 837                 {
 838                 case 1:
 839                   /* A B A */
 840                   to_next[-2] = pi1;
 841                   to_next -= 1;
 842                   n_left_to_next += 1;
 843                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 844                   break;
 845
 846                 case 2:
 847                   /* A A B */
 848                   to_next -= 1;
 849                   n_left_to_next += 1;
 850                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 851                   break;
 852
 853                 case 3:
 854                   /* A B C */
 855                   to_next -= 2;
 856                   n_left_to_next += 2;
 857                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 858                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 859                   if (next0 == next1)
 860                     {
 861                       /* A B B */
 862                       vlib_put_next_frame (vm, node, next, n_left_to_next);
 863                       next = next1;
 864                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 865                     }
 866                 }
 867             }
 868         }
 869
 870       while (n_left_from > 0 && n_left_to_next > 0)
 871         {
 872           vlib_buffer_t * p0;
 873           ip4_header_t * ip0;
 874           __attribute__((unused)) tcp_header_t * tcp0;
 875           ip_lookup_next_t next0;
 876           ip_adjacency_t * adj0;
 877           ip4_fib_mtrie_t * mtrie0;
 878           ip4_fib_mtrie_leaf_t leaf0;
 879           ip4_address_t * dst_addr0;
 880           __attribute__((unused)) u32 pi0, fib_index0, adj_index0, is_tcp_udp0;
 881           u32 flow_hash_config0, hash_c0;
 882
 883           pi0 = from[0];
 884           to_next[0] = pi0;
 885
 886           p0 = vlib_get_buffer (vm, pi0);
 887
 888           ip0 = vlib_buffer_get_current (p0);
 889
 890           if (is_indirect)
 891             {
 892               ip_adjacency_t * iadj0;
 893               iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
 894               dst_addr0 = &iadj0->indirect.next_hop.ip4;
 895             }
 896           else
 897             {
 898               dst_addr0 = &ip0->dst_address;
 899             }
 900
 901           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 902           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 903             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 904
 905           if (! lookup_for_responses_to_locally_received_packets)
 906             {
 907               mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
 908
 909               leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
 910
 911               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 0);
 912             }
 913
 914           tcp0 = (void *) (ip0 + 1);
 915
 916           is_tcp_udp0 = (ip0->protocol == IP_PROTOCOL_TCP
 917                          || ip0->protocol == IP_PROTOCOL_UDP);
 918
 919           if (! lookup_for_responses_to_locally_received_packets)
 920             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
 921
 922           if (! lookup_for_responses_to_locally_received_packets)
 923             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 924
 925           if (! lookup_for_responses_to_locally_received_packets)
 926             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 927
 928           if (lookup_for_responses_to_locally_received_packets)
 929             adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 930           else
 931             {
 932               /* Handle default route. */
 933               leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 934               adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 935             }
 936
 937           ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
 938                                                            dst_addr0,
 939                                                            /* no_default_route */ 0));
 940
 941           adj0 = ip_get_adjacency (lm, adj_index0);
 942
 943           next0 = adj0->lookup_next_index;
 944
 945           /* Use flow hash to compute multipath adjacency. */
 946           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 947           if (PREDICT_FALSE(adj0->n_adj > 1))
 948             {
 949               flow_hash_config0 =
 950                 vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
 951
 952               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 953                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 954             }
 955
 956           ASSERT (adj0->n_adj > 0);
 957           ASSERT (is_pow2 (adj0->n_adj));
 958           adj_index0 += (hash_c0 & (adj0->n_adj - 1));
 959
 960           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 961
 962           vlib_increment_combined_counter
 963               (cm, cpu_index, adj_index0, 1,
 964                vlib_buffer_length_in_chain (vm, p0)
 965                + sizeof(ethernet_header_t));
 966
 967           from += 1;
 968           to_next += 1;
 969           n_left_to_next -= 1;
 970           n_left_from -= 1;
 971
 972           if (PREDICT_FALSE (next0 != next))
 973             {
 974               n_left_to_next += 1;
 975               vlib_put_next_frame (vm, node, next, n_left_to_next);
 976               next = next0;
 977               vlib_get_next_frame (vm, node, next,
 978                                    to_next, n_left_to_next);
 979               to_next[0] = pi0;
 980               to_next += 1;
 981               n_left_to_next -= 1;
 982             }
 983         }
 984
 985       vlib_put_next_frame (vm, node, next, n_left_to_next);
 986     }
 987
 988   if (node->flags & VLIB_NODE_FLAG_TRACE)
 989     ip4_forward_next_trace(vm, node, frame, VLIB_TX);
 990
 991   return frame->n_vectors;
 992 }
 993
 994 static uword
 995 ip4_lookup (vlib_main_t * vm,
 996             vlib_node_runtime_t * node,
 997             vlib_frame_t * frame)
 998 {
 999   return ip4_lookup_inline (vm, node, frame,
1000                             /* lookup_for_responses_to_locally_received_packets */ 0,
1001                             /* is_indirect */ 0);
1002
1003 }
1004
1005 void ip4_adjacency_set_interface_route (vnet_main_t * vnm,
1006                                         ip_adjacency_t * adj,
1007                                         u32 sw_if_index,
1008                                         u32 if_address_index)
1009 {
1010   vnet_hw_interface_t * hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
1011   ip_lookup_next_t n;
1012   vnet_l3_packet_type_t packet_type;
1013   u32 node_index;
1014
1015   if (hw->hw_class_index == ethernet_hw_interface_class.index
1016       || hw->hw_class_index == srp_hw_interface_class.index)
1017     {
1018       /*
1019        * We have a bit of a problem in this case. ip4-arp uses
1020        * the rewrite_header.next_index to hand pkts to the
1021        * indicated inteface output node. We can end up in
1022        * ip4_rewrite_local, too, which also pays attention to
1023        * rewrite_header.next index. Net result: a hack in
1024        * ip4_rewrite_local...
1025        */
1026       n = IP_LOOKUP_NEXT_ARP;
1027       node_index = ip4_arp_node.index;
1028       adj->if_address_index = if_address_index;
1029       adj->arp.next_hop.ip4.as_u32 = 0;
1030       ip46_address_reset(&adj->arp.next_hop);
1031       packet_type = VNET_L3_PACKET_TYPE_ARP;
1032     }
1033   else
1034     {
1035       n = IP_LOOKUP_NEXT_REWRITE;
1036       node_index = ip4_rewrite_node.index;
1037       packet_type = VNET_L3_PACKET_TYPE_IP4;
1038     }
1039
1040   adj->lookup_next_index = n;
1041   vnet_rewrite_for_sw_interface
1042     (vnm,
1043      packet_type,
1044      sw_if_index,
1045      node_index,
1046      VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST,
1047      &adj->rewrite_header,
1048      sizeof (adj->rewrite_data));
1049 }
1050
1051 static void
1052 ip4_add_interface_routes (u32 sw_if_index,
1053                           ip4_main_t * im, u32 fib_index,
1054                           ip_interface_address_t * a)
1055 {
1056   vnet_main_t * vnm = vnet_get_main();
1057   ip_lookup_main_t * lm = &im->lookup_main;
1058   ip_adjacency_t * adj;
1059   ip4_address_t * address = ip_interface_address_get_address (lm, a);
1060   ip4_add_del_route_args_t x;
1061   vnet_hw_interface_t * hw_if = vnet_get_sup_hw_interface (vnm, sw_if_index);
1062   u32 classify_table_index;
1063
1064   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1065   x.table_index_or_table_id = fib_index;
1066   x.flags = (IP4_ROUTE_FLAG_ADD
1067              | IP4_ROUTE_FLAG_FIB_INDEX
1068              | IP4_ROUTE_FLAG_NO_REDISTRIBUTE);
1069   x.dst_address = address[0];
1070   x.dst_address_length = a->address_length;
1071   x.n_add_adj = 0;
1072   x.add_adj = 0;
1073
1074   a->neighbor_probe_adj_index = ~0;
1075   if (a->address_length < 32)
1076     {
1077       adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1078                               &x.adj_index);
1079       ip4_adjacency_set_interface_route (vnm, adj, sw_if_index, a - lm->if_address_pool);
1080       ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1081       ip4_add_del_route (im, &x);
1082       a->neighbor_probe_adj_index = x.adj_index;
1083     }
1084
1085   /* Add e.g. 1.1.1.1/32 as local to this host. */
1086   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1087                           &x.adj_index);
1088
1089   classify_table_index = ~0;
1090   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
1091     classify_table_index = lm->classify_table_index_by_sw_if_index [sw_if_index];
1092   if (classify_table_index != (u32) ~0)
1093     {
1094       adj->lookup_next_index = IP_LOOKUP_NEXT_CLASSIFY;
1095       adj->classify.table_index = classify_table_index;
1096     }
1097   else
1098     adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
1099
1100   adj->if_address_index = a - lm->if_address_pool;
1101   adj->rewrite_header.sw_if_index = sw_if_index;
1102   adj->rewrite_header.max_l3_packet_bytes = hw_if->max_l3_packet_bytes[VLIB_RX];
1103   /*
1104    * Local adjs are never to be rewritten. Spoofed pkts w/ src = dst = local
1105    * fail an RPF-ish check, but still go thru the rewrite code...
1106    */
1107   adj->rewrite_header.data_bytes = 0;
1108
1109   ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1110   x.dst_address_length = 32;
1111   ip4_add_del_route (im, &x);
1112 }
1113
1114 static void
1115 ip4_del_interface_routes (ip4_main_t * im, u32 fib_index, ip4_address_t * address, u32 address_length)
1116 {
1117   ip4_add_del_route_args_t x;
1118
1119   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1120   x.table_index_or_table_id = fib_index;
1121   x.flags = (IP4_ROUTE_FLAG_DEL
1122              | IP4_ROUTE_FLAG_FIB_INDEX
1123              | IP4_ROUTE_FLAG_NO_REDISTRIBUTE);
1124   x.dst_address = address[0];
1125   x.dst_address_length = address_length;
1126   x.adj_index = ~0;
1127   x.n_add_adj = 0;
1128   x.add_adj = 0;
1129
1130   if (address_length < 32)
1131     ip4_add_del_route (im, &x);
1132
1133   x.dst_address_length = 32;
1134   ip4_add_del_route (im, &x);
1135
1136   ip4_delete_matching_routes (im,
1137                               fib_index,
1138                               IP4_ROUTE_FLAG_FIB_INDEX,
1139                               address,
1140                               address_length);
1141 }
1142
1143 typedef struct {
1144     u32 sw_if_index;
1145     ip4_address_t address;
1146     u32 length;
1147 } ip4_interface_address_t;
1148
1149 static clib_error_t *
1150 ip4_add_del_interface_address_internal (vlib_main_t * vm,
1151                                         u32 sw_if_index,
1152                                         ip4_address_t * new_address,
1153                                         u32 new_length,
1154                                         u32 redistribute,
1155                                         u32 insert_routes,
1156                                         u32 is_del);
1157
1158 static clib_error_t *
1159 ip4_add_del_interface_address_internal (vlib_main_t * vm,
1160                                         u32 sw_if_index,
1161                                         ip4_address_t * address,
1162                                         u32 address_length,
1163                                         u32 redistribute,
1164                                         u32 insert_routes,
1165                                         u32 is_del)
1166 {
1167   vnet_main_t * vnm = vnet_get_main();
1168   ip4_main_t * im = &ip4_main;
1169   ip_lookup_main_t * lm = &im->lookup_main;
1170   clib_error_t * error = 0;
1171   u32 if_address_index, elts_before;
1172   ip4_address_fib_t ip4_af, * addr_fib = 0;
1173
1174   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1175   ip4_addr_fib_init (&ip4_af, address,
1176                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
1177   vec_add1 (addr_fib, ip4_af);
1178
1179   /* When adding an address check that it does not conflict with an existing address. */
1180   if (! is_del)
1181     {
1182       ip_interface_address_t * ia;
1183       foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
1184                                     0 /* honor unnumbered */,
1185       ({
1186         ip4_address_t * x = ip_interface_address_get_address (&im->lookup_main, ia);
1187
1188         if (ip4_destination_matches_route (im, address, x, ia->address_length)
1189             || ip4_destination_matches_route (im, x, address, address_length))
1190           return clib_error_create ("failed to add %U which conflicts with %U for interface %U",
1191                                     format_ip4_address_and_length, address, address_length,
1192                                     format_ip4_address_and_length, x, ia->address_length,
1193                                     format_vnet_sw_if_index_name, vnm, sw_if_index);
1194       }));
1195     }
1196
1197   elts_before = pool_elts (lm->if_address_pool);
1198
1199   error = ip_interface_address_add_del
1200     (lm,
1201      sw_if_index,
1202      addr_fib,
1203      address_length,
1204      is_del,
1205      &if_address_index);
1206   if (error)
1207     goto done;
1208
1209   if (vnet_sw_interface_is_admin_up (vnm, sw_if_index) && insert_routes)
1210     {
1211       if (is_del)
1212         ip4_del_interface_routes (im, ip4_af.fib_index, address,
1213                                   address_length);
1214
1215       else
1216           ip4_add_interface_routes (sw_if_index,
1217                                     im, ip4_af.fib_index,
1218                                     pool_elt_at_index
1219                                     (lm->if_address_pool, if_address_index));
1220     }
1221
1222   /* If pool did not grow/shrink: add duplicate address. */
1223   if (elts_before != pool_elts (lm->if_address_pool))
1224     {
1225       ip4_add_del_interface_address_callback_t * cb;
1226       vec_foreach (cb, im->add_del_interface_address_callbacks)
1227         cb->function (im, cb->function_opaque, sw_if_index,
1228                       address, address_length,
1229                       if_address_index,
1230                       is_del);
1231     }
1232
1233  done:
1234   vec_free (addr_fib);
1235   return error;
1236 }
1237
1238 clib_error_t *
1239 ip4_add_del_interface_address (vlib_main_t * vm, u32 sw_if_index,
1240                                ip4_address_t * address, u32 address_length,
1241                                u32 is_del)
1242 {
1243   return ip4_add_del_interface_address_internal
1244     (vm, sw_if_index, address, address_length,
1245      /* redistribute */ 1,
1246      /* insert_routes */ 1,
1247      is_del);
1248 }
1249
1250 static clib_error_t *
1251 ip4_sw_interface_admin_up_down (vnet_main_t * vnm,
1252                                 u32 sw_if_index,
1253                                 u32 flags)
1254 {
1255   ip4_main_t * im = &ip4_main;
1256   ip_interface_address_t * ia;
1257   ip4_address_t * a;
1258   u32 is_admin_up, fib_index;
1259
1260   /* Fill in lookup tables with default table (0). */
1261   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1262
1263   vec_validate_init_empty (im->lookup_main.if_address_pool_index_by_sw_if_index, sw_if_index, ~0);
1264
1265   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1266
1267   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
1268
1269   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
1270                                 0 /* honor unnumbered */,
1271   ({
1272     a = ip_interface_address_get_address (&im->lookup_main, ia);
1273     if (is_admin_up)
1274       ip4_add_interface_routes (sw_if_index,
1275                                 im, fib_index,
1276                                 ia);
1277     else
1278       ip4_del_interface_routes (im, fib_index,
1279                                 a, ia->address_length);
1280   }));
1281
1282   return 0;
1283 }
1284
1285 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip4_sw_interface_admin_up_down);
1286
1287 static clib_error_t *
1288 ip4_sw_interface_add_del (vnet_main_t * vnm,
1289                           u32 sw_if_index,
1290                           u32 is_add)
1291 {
1292   vlib_main_t * vm = vnm->vlib_main;
1293   ip4_main_t * im = &ip4_main;
1294   ip_lookup_main_t * lm = &im->lookup_main;
1295   u32 ci, cast;
1296
1297   for (cast = 0; cast < VNET_N_CAST; cast++)
1298     {
1299       ip_config_main_t * cm = &lm->rx_config_mains[cast];
1300       vnet_config_main_t * vcm = &cm->config_main;
1301
1302       if (! vcm->node_index_by_feature_index)
1303         {
1304           if (cast == VNET_UNICAST)
1305             {
1306               static char * start_nodes[] = { "ip4-input", "ip4-input-no-checksum", };
1307               static char * feature_nodes[] = {
1308                 [IP4_RX_FEATURE_CHECK_ACCESS] = "ip4-inacl",
1309                 [IP4_RX_FEATURE_SOURCE_CHECK_REACHABLE_VIA_RX] = "ip4-source-check-via-rx",
1310                 [IP4_RX_FEATURE_SOURCE_CHECK_REACHABLE_VIA_ANY] = "ip4-source-check-via-any",
1311                 [IP4_RX_FEATURE_IPSEC] = "ipsec-input-ip4",
1312                 [IP4_RX_FEATURE_VPATH] = "vpath-input-ip4",
1313                 [IP4_RX_FEATURE_LOOKUP] = "ip4-lookup",
1314               };
1315
1316               vnet_config_init (vm, vcm,
1317                                 start_nodes, ARRAY_LEN (start_nodes),
1318                                 feature_nodes, ARRAY_LEN (feature_nodes));
1319             }
1320           else
1321             {
1322               static char * start_nodes[] = { "ip4-input", "ip4-input-no-checksum", };
1323               static char * feature_nodes[] = {
1324                 [IP4_RX_FEATURE_VPATH] = "vpath-input-ip4",
1325                 [IP4_RX_FEATURE_LOOKUP] = "ip4-lookup-multicast",
1326               };
1327
1328               vnet_config_init (vm, vcm,
1329                                 start_nodes, ARRAY_LEN (start_nodes),
1330                                 feature_nodes, ARRAY_LEN (feature_nodes));
1331             }
1332         }
1333
1334       vec_validate_init_empty (cm->config_index_by_sw_if_index, sw_if_index, ~0);
1335       ci = cm->config_index_by_sw_if_index[sw_if_index];
1336
1337       if (is_add)
1338         ci = vnet_config_add_feature (vm, vcm,
1339                                       ci,
1340                                       IP4_RX_FEATURE_LOOKUP,
1341                                       /* config data */ 0,
1342                                       /* # bytes of config data */ 0);
1343       else
1344         ci = vnet_config_del_feature (vm, vcm,
1345                                       ci,
1346                                       IP4_RX_FEATURE_LOOKUP,
1347                                       /* config data */ 0,
1348                                       /* # bytes of config data */ 0);
1349
1350       cm->config_index_by_sw_if_index[sw_if_index] = ci;
1351     }
1352
1353   return /* no error */ 0;
1354 }
1355
1356 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
1357
1358 static u8 * format_ip4_lookup_trace (u8 * s, va_list * args);
1359
1360 VLIB_REGISTER_NODE (ip4_lookup_node) = {
1361   .function = ip4_lookup,
1362   .name = "ip4-lookup",
1363   .vector_size = sizeof (u32),
1364
1365   .format_trace = format_ip4_lookup_trace,
1366
1367   .n_next_nodes = IP_LOOKUP_N_NEXT,
1368   .next_nodes = IP4_LOOKUP_NEXT_NODES,
1369 };
1370
1371 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup)
1372
1373 static uword
1374 ip4_indirect (vlib_main_t * vm,
1375                vlib_node_runtime_t * node,
1376                vlib_frame_t * frame)
1377 {
1378   return ip4_lookup_inline (vm, node, frame,
1379                             /* lookup_for_responses_to_locally_received_packets */ 0,
1380                             /* is_indirect */ 1);
1381 }
1382
1383 VLIB_REGISTER_NODE (ip4_indirect_node) = {
1384   .function = ip4_indirect,
1385   .name = "ip4-indirect",
1386   .vector_size = sizeof (u32),
1387
1388   .format_trace = format_ip4_lookup_trace,
1389
1390   .n_next_nodes = IP_LOOKUP_N_NEXT,
1391   .next_nodes = IP4_LOOKUP_NEXT_NODES,
1392 };
1393
1394 VLIB_NODE_FUNCTION_MULTIARCH (ip4_indirect_node, ip4_indirect)
1395
1396
1397 /* Global IP4 main. */
1398 ip4_main_t ip4_main;
1399
1400 clib_error_t *
1401 ip4_lookup_init (vlib_main_t * vm)
1402 {
1403   ip4_main_t * im = &ip4_main;
1404   uword i;
1405
1406   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
1407     {
1408       u32 m;
1409
1410       if (i < 32)
1411         m = pow2_mask (i) << (32 - i);
1412       else
1413         m = ~0;
1414       im->fib_masks[i] = clib_host_to_net_u32 (m);
1415     }
1416
1417   /* Create FIB with index 0 and table id of 0. */
1418   find_ip4_fib_by_table_index_or_id (im, /* table id */ 0, IP4_ROUTE_FLAG_TABLE_ID);
1419
1420   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
1421
1422   {
1423     pg_node_t * pn;
1424     pn = pg_get_node (ip4_lookup_node.index);
1425     pn->unformat_edit = unformat_pg_ip4_header;
1426   }
1427
1428   {
1429     ethernet_arp_header_t h;
1430
1431     memset (&h, 0, sizeof (h));
1432
1433     /* Set target ethernet address to all zeros. */
1434     memset (h.ip4_over_ethernet[1].ethernet, 0, sizeof (h.ip4_over_ethernet[1].ethernet));
1435
1436 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
1437 #define _8(f,v) h.f = v;
1438     _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
1439     _16 (l3_type, ETHERNET_TYPE_IP4);
1440     _8 (n_l2_address_bytes, 6);
1441     _8 (n_l3_address_bytes, 4);
1442     _16 (opcode, ETHERNET_ARP_OPCODE_request);
1443 #undef _16
1444 #undef _8
1445
1446     vlib_packet_template_init (vm,
1447                                &im->ip4_arp_request_packet_template,
1448                                /* data */ &h,
1449                                sizeof (h),
1450                                /* alloc chunk size */ 8,
1451                                "ip4 arp");
1452   }
1453
1454   return 0;
1455 }
1456
1457 VLIB_INIT_FUNCTION (ip4_lookup_init);
1458
1459 typedef struct {
1460   /* Adjacency taken. */
1461   u32 adj_index;
1462   u32 flow_hash;
1463   u32 fib_index;
1464
1465   /* Packet data, possibly *after* rewrite. */
1466   u8 packet_data[64 - 1*sizeof(u32)];
1467 } ip4_forward_next_trace_t;
1468
1469 static u8 * format_ip4_forward_next_trace (u8 * s, va_list * args)
1470 {
1471   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1472   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1473   ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1474   uword indent = format_get_indent (s);
1475   s = format (s, "%U%U",
1476                 format_white_space, indent,
1477                 format_ip4_header, t->packet_data);
1478   return s;
1479 }
1480
1481 static u8 * format_ip4_lookup_trace (u8 * s, va_list * args)
1482 {
1483   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1484   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1485   ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1486   vnet_main_t * vnm = vnet_get_main();
1487   ip4_main_t * im = &ip4_main;
1488   uword indent = format_get_indent (s);
1489
1490   s = format (s, "fib %d adj-idx %d : %U flow hash: 0x%08x",
1491               t->fib_index, t->adj_index, format_ip_adjacency,
1492               vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1493   s = format (s, "\n%U%U",
1494               format_white_space, indent,
1495               format_ip4_header, t->packet_data);
1496   return s;
1497 }
1498
1499 static u8 * format_ip4_rewrite_trace (u8 * s, va_list * args)
1500 {
1501   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1502   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1503   ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1504   vnet_main_t * vnm = vnet_get_main();
1505   ip4_main_t * im = &ip4_main;
1506   uword indent = format_get_indent (s);
1507
1508   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
1509               t->fib_index, t->adj_index, format_ip_adjacency,
1510               vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1511   s = format (s, "\n%U%U",
1512               format_white_space, indent,
1513               format_ip_adjacency_packet_data,
1514               vnm, &im->lookup_main, t->adj_index,
1515               t->packet_data, sizeof (t->packet_data));
1516   return s;
1517 }
1518
1519 /* Common trace function for all ip4-forward next nodes. */
1520 void
1521 ip4_forward_next_trace (vlib_main_t * vm,
1522                         vlib_node_runtime_t * node,
1523                         vlib_frame_t * frame,
1524                         vlib_rx_or_tx_t which_adj_index)
1525 {
1526   u32 * from, n_left;
1527   ip4_main_t * im = &ip4_main;
1528
1529   n_left = frame->n_vectors;
1530   from = vlib_frame_vector_args (frame);
1531
1532   while (n_left >= 4)
1533     {
1534       u32 bi0, bi1;
1535       vlib_buffer_t * b0, * b1;
1536       ip4_forward_next_trace_t * t0, * t1;
1537
1538       /* Prefetch next iteration. */
1539       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1540       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1541
1542       bi0 = from[0];
1543       bi1 = from[1];
1544
1545       b0 = vlib_get_buffer (vm, bi0);
1546       b1 = vlib_get_buffer (vm, bi1);
1547
1548       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1549         {
1550           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1551           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1552           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1553           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1554               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1555               vec_elt (im->fib_index_by_sw_if_index,
1556                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1557
1558           clib_memcpy (t0->packet_data,
1559                   vlib_buffer_get_current (b0),
1560                   sizeof (t0->packet_data));
1561         }
1562       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1563         {
1564           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1565           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1566           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1567           t1->fib_index = (vnet_buffer(b1)->sw_if_index[VLIB_TX] != (u32)~0) ?
1568               vnet_buffer(b1)->sw_if_index[VLIB_TX] :
1569               vec_elt (im->fib_index_by_sw_if_index,
1570                        vnet_buffer(b1)->sw_if_index[VLIB_RX]);
1571           clib_memcpy (t1->packet_data,
1572                   vlib_buffer_get_current (b1),
1573                   sizeof (t1->packet_data));
1574         }
1575       from += 2;
1576       n_left -= 2;
1577     }
1578
1579   while (n_left >= 1)
1580     {
1581       u32 bi0;
1582       vlib_buffer_t * b0;
1583       ip4_forward_next_trace_t * t0;
1584
1585       bi0 = from[0];
1586
1587       b0 = vlib_get_buffer (vm, bi0);
1588
1589       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1590         {
1591           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1592           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1593           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1594           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1595               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1596               vec_elt (im->fib_index_by_sw_if_index,
1597                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1598           clib_memcpy (t0->packet_data,
1599                   vlib_buffer_get_current (b0),
1600                   sizeof (t0->packet_data));
1601         }
1602       from += 1;
1603       n_left -= 1;
1604     }
1605 }
1606
1607 static uword
1608 ip4_drop_or_punt (vlib_main_t * vm,
1609                   vlib_node_runtime_t * node,
1610                   vlib_frame_t * frame,
1611                   ip4_error_t error_code)
1612 {
1613   u32 * buffers = vlib_frame_vector_args (frame);
1614   uword n_packets = frame->n_vectors;
1615
1616   vlib_error_drop_buffers (vm, node,
1617                            buffers,
1618                            /* stride */ 1,
1619                            n_packets,
1620                            /* next */ 0,
1621                            ip4_input_node.index,
1622                            error_code);
1623
1624   if (node->flags & VLIB_NODE_FLAG_TRACE)
1625     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1626
1627   return n_packets;
1628 }
1629
1630 static uword
1631 ip4_drop (vlib_main_t * vm,
1632           vlib_node_runtime_t * node,
1633           vlib_frame_t * frame)
1634 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_DROP); }
1635
1636 static uword
1637 ip4_punt (vlib_main_t * vm,
1638           vlib_node_runtime_t * node,
1639           vlib_frame_t * frame)
1640 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_PUNT); }
1641
1642 static uword
1643 ip4_miss (vlib_main_t * vm,
1644           vlib_node_runtime_t * node,
1645           vlib_frame_t * frame)
1646 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_DST_LOOKUP_MISS); }
1647
1648 VLIB_REGISTER_NODE (ip4_drop_node,static) = {
1649   .function = ip4_drop,
1650   .name = "ip4-drop",
1651   .vector_size = sizeof (u32),
1652
1653   .format_trace = format_ip4_forward_next_trace,
1654
1655   .n_next_nodes = 1,
1656   .next_nodes = {
1657     [0] = "error-drop",
1658   },
1659 };
1660
1661 VLIB_NODE_FUNCTION_MULTIARCH (ip4_drop_node, ip4_drop)
1662
1663 VLIB_REGISTER_NODE (ip4_punt_node,static) = {
1664   .function = ip4_punt,
1665   .name = "ip4-punt",
1666   .vector_size = sizeof (u32),
1667
1668   .format_trace = format_ip4_forward_next_trace,
1669
1670   .n_next_nodes = 1,
1671   .next_nodes = {
1672     [0] = "error-punt",
1673   },
1674 };
1675
1676 VLIB_NODE_FUNCTION_MULTIARCH (ip4_punt_node, ip4_punt)
1677
1678 VLIB_REGISTER_NODE (ip4_miss_node,static) = {
1679   .function = ip4_miss,
1680   .name = "ip4-miss",
1681   .vector_size = sizeof (u32),
1682
1683   .format_trace = format_ip4_forward_next_trace,
1684
1685   .n_next_nodes = 1,
1686   .next_nodes = {
1687     [0] = "error-drop",
1688   },
1689 };
1690
1691 VLIB_NODE_FUNCTION_MULTIARCH (ip4_miss_node, ip4_miss)
1692
1693 /* Compute TCP/UDP/ICMP4 checksum in software. */
1694 u16
1695 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1696                               ip4_header_t * ip0)
1697 {
1698   ip_csum_t sum0;
1699   u32 ip_header_length, payload_length_host_byte_order;
1700   u32 n_this_buffer, n_bytes_left;
1701   u16 sum16;
1702   void * data_this_buffer;
1703
1704   /* Initialize checksum with ip header. */
1705   ip_header_length = ip4_header_bytes (ip0);
1706   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->length) - ip_header_length;
1707   sum0 = clib_host_to_net_u32 (payload_length_host_byte_order + (ip0->protocol << 16));
1708
1709   if (BITS (uword) == 32)
1710     {
1711       sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u32));
1712       sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->dst_address, u32));
1713     }
1714   else
1715     sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1716
1717   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1718   data_this_buffer = (void *) ip0 + ip_header_length;
1719   if (n_this_buffer + ip_header_length > p0->current_length)
1720     n_this_buffer = p0->current_length > ip_header_length ? p0->current_length - ip_header_length : 0;
1721   while (1)
1722     {
1723       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1724       n_bytes_left -= n_this_buffer;
1725       if (n_bytes_left == 0)
1726         break;
1727
1728       ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1729       p0 = vlib_get_buffer (vm, p0->next_buffer);
1730       data_this_buffer = vlib_buffer_get_current (p0);
1731       n_this_buffer = p0->current_length;
1732     }
1733
1734   sum16 = ~ ip_csum_fold (sum0);
1735
1736   return sum16;
1737 }
1738
1739 static u32
1740 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1741 {
1742   ip4_header_t * ip0 = vlib_buffer_get_current (p0);
1743   udp_header_t * udp0;
1744   u16 sum16;
1745
1746   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1747           || ip0->protocol == IP_PROTOCOL_UDP);
1748
1749   udp0 = (void *) (ip0 + 1);
1750   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1751     {
1752       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1753                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1754       return p0->flags;
1755     }
1756
1757   sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1758
1759   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1760                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1761
1762   return p0->flags;
1763 }
1764
1765 static uword
1766 ip4_local (vlib_main_t * vm,
1767            vlib_node_runtime_t * node,
1768            vlib_frame_t * frame)
1769 {
1770   ip4_main_t * im = &ip4_main;
1771   ip_lookup_main_t * lm = &im->lookup_main;
1772   ip_local_next_t next_index;
1773   u32 * from, * to_next, n_left_from, n_left_to_next;
1774   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip4_input_node.index);
1775
1776   from = vlib_frame_vector_args (frame);
1777   n_left_from = frame->n_vectors;
1778   next_index = node->cached_next_index;
1779
1780   if (node->flags & VLIB_NODE_FLAG_TRACE)
1781     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1782
1783   while (n_left_from > 0)
1784     {
1785       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1786
1787       while (n_left_from >= 4 && n_left_to_next >= 2)
1788         {
1789           vlib_buffer_t * p0, * p1;
1790           ip4_header_t * ip0, * ip1;
1791           udp_header_t * udp0, * udp1;
1792           ip4_fib_mtrie_t * mtrie0, * mtrie1;
1793           ip4_fib_mtrie_leaf_t leaf0, leaf1;
1794           ip_adjacency_t * adj0, * adj1;
1795           u32 pi0, ip_len0, udp_len0, flags0, next0, fib_index0, adj_index0;
1796           u32 pi1, ip_len1, udp_len1, flags1, next1, fib_index1, adj_index1;
1797           i32 len_diff0, len_diff1;
1798           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1799           u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1800           u8 enqueue_code;
1801
1802           pi0 = to_next[0] = from[0];
1803           pi1 = to_next[1] = from[1];
1804           from += 2;
1805           n_left_from -= 2;
1806           to_next += 2;
1807           n_left_to_next -= 2;
1808
1809           p0 = vlib_get_buffer (vm, pi0);
1810           p1 = vlib_get_buffer (vm, pi1);
1811
1812           ip0 = vlib_buffer_get_current (p0);
1813           ip1 = vlib_buffer_get_current (p1);
1814
1815           fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
1816                                 vnet_buffer(p0)->sw_if_index[VLIB_RX]);
1817           fib_index1 = vec_elt (im->fib_index_by_sw_if_index,
1818                                 vnet_buffer(p1)->sw_if_index[VLIB_RX]);
1819
1820           mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
1821           mtrie1 = &vec_elt_at_index (im->fibs, fib_index1)->mtrie;
1822
1823           leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
1824
1825           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 0);
1826           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 0);
1827
1828           /* Treat IP frag packets as "experimental" protocol for now
1829              until support of IP frag reassembly is implemented */
1830           proto0 = ip4_is_fragment(ip0) ? 0xfe : ip0->protocol;
1831           proto1 = ip4_is_fragment(ip1) ? 0xfe : ip1->protocol;
1832           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1833           is_udp1 = proto1 == IP_PROTOCOL_UDP;
1834           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1835           is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1836
1837           flags0 = p0->flags;
1838           flags1 = p1->flags;
1839
1840           good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1841           good_tcp_udp1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1842
1843           udp0 = ip4_next_header (ip0);
1844           udp1 = ip4_next_header (ip1);
1845
1846           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1847           good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1848           good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1849
1850           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
1851           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 1);
1852
1853           /* Verify UDP length. */
1854           ip_len0 = clib_net_to_host_u16 (ip0->length);
1855           ip_len1 = clib_net_to_host_u16 (ip1->length);
1856           udp_len0 = clib_net_to_host_u16 (udp0->length);
1857           udp_len1 = clib_net_to_host_u16 (udp1->length);
1858
1859           len_diff0 = ip_len0 - udp_len0;
1860           len_diff1 = ip_len1 - udp_len1;
1861
1862           len_diff0 = is_udp0 ? len_diff0 : 0;
1863           len_diff1 = is_udp1 ? len_diff1 : 0;
1864
1865           if (PREDICT_FALSE (! (is_tcp_udp0 & is_tcp_udp1
1866                                 & good_tcp_udp0 & good_tcp_udp1)))
1867             {
1868               if (is_tcp_udp0)
1869                 {
1870                   if (is_tcp_udp0
1871                       && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1872                     flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
1873                   good_tcp_udp0 =
1874                     (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1875                   good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1876                 }
1877               if (is_tcp_udp1)
1878                 {
1879                   if (is_tcp_udp1
1880                       && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1881                     flags1 = ip4_tcp_udp_validate_checksum (vm, p1);
1882                   good_tcp_udp1 =
1883                     (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1884                   good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1885                 }
1886             }
1887
1888           good_tcp_udp0 &= len_diff0 >= 0;
1889           good_tcp_udp1 &= len_diff1 >= 0;
1890
1891           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1892           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 2);
1893
1894           error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1895
1896           error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
1897           error1 = len_diff1 < 0 ? IP4_ERROR_UDP_LENGTH : error1;
1898
1899           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1900           error0 = (is_tcp_udp0 && ! good_tcp_udp0
1901                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0
1902                     : error0);
1903           error1 = (is_tcp_udp1 && ! good_tcp_udp1
1904                     ? IP4_ERROR_TCP_CHECKSUM + is_udp1
1905                     : error1);
1906
1907           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1908           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 3);
1909
1910           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1911           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
1912
1913           vnet_buffer (p1)->ip.adj_index[VLIB_RX] = adj_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
1914           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
1915
1916           ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
1917                                                            &ip0->src_address,
1918                                                            /* no_default_route */ 1));
1919           ASSERT (adj_index1 == ip4_fib_lookup_with_table (im, fib_index1,
1920                                                            &ip1->src_address,
1921                                                            /* no_default_route */ 1));
1922
1923           adj0 = ip_get_adjacency (lm, adj_index0);
1924           adj1 = ip_get_adjacency (lm, adj_index1);
1925
1926           /*
1927            * Must have a route to source otherwise we drop the packet.
1928            * ip4 broadcasts are accepted, e.g. to make dhcp client work
1929            */
1930           error0 = (error0 == IP4_ERROR_UNKNOWN_PROTOCOL
1931                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_REWRITE
1932                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP
1933                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_LOCAL
1934                     && ip0->dst_address.as_u32 != 0xFFFFFFFF
1935                     ? IP4_ERROR_SRC_LOOKUP_MISS
1936                     : error0);
1937           error1 = (error1 == IP4_ERROR_UNKNOWN_PROTOCOL
1938                     && adj1->lookup_next_index != IP_LOOKUP_NEXT_REWRITE
1939                     && adj1->lookup_next_index != IP_LOOKUP_NEXT_ARP
1940                     && adj1->lookup_next_index != IP_LOOKUP_NEXT_LOCAL
1941                     && ip0->dst_address.as_u32 != 0xFFFFFFFF
1942                     ? IP4_ERROR_SRC_LOOKUP_MISS
1943                     : error1);
1944
1945           next0 = lm->local_next_by_ip_protocol[proto0];
1946           next1 = lm->local_next_by_ip_protocol[proto1];
1947
1948           next0 = error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1949           next1 = error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1950
1951           p0->error = error0 ? error_node->errors[error0] : 0;
1952           p1->error = error1 ? error_node->errors[error1] : 0;
1953
1954           enqueue_code = (next0 != next_index) + 2*(next1 != next_index);
1955
1956           if (PREDICT_FALSE (enqueue_code != 0))
1957             {
1958               switch (enqueue_code)
1959                 {
1960                 case 1:
1961                   /* A B A */
1962                   to_next[-2] = pi1;
1963                   to_next -= 1;
1964                   n_left_to_next += 1;
1965                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
1966                   break;
1967
1968                 case 2:
1969                   /* A A B */
1970                   to_next -= 1;
1971                   n_left_to_next += 1;
1972                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
1973                   break;
1974
1975                 case 3:
1976                   /* A B B or A B C */
1977                   to_next -= 2;
1978                   n_left_to_next += 2;
1979                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
1980                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
1981                   if (next0 == next1)
1982                     {
1983                       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1984                       next_index = next1;
1985                       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1986                     }
1987                   break;
1988                 }
1989             }
1990         }
1991
1992       while (n_left_from > 0 && n_left_to_next > 0)
1993         {
1994           vlib_buffer_t * p0;
1995           ip4_header_t * ip0;
1996           udp_header_t * udp0;
1997           ip4_fib_mtrie_t * mtrie0;
1998           ip4_fib_mtrie_leaf_t leaf0;
1999           ip_adjacency_t * adj0;
2000           u32 pi0, next0, ip_len0, udp_len0, flags0, fib_index0, adj_index0;
2001           i32 len_diff0;
2002           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
2003
2004           pi0 = to_next[0] = from[0];
2005           from += 1;
2006           n_left_from -= 1;
2007           to_next += 1;
2008           n_left_to_next -= 1;
2009
2010           p0 = vlib_get_buffer (vm, pi0);
2011
2012           ip0 = vlib_buffer_get_current (p0);
2013
2014           fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
2015                                 vnet_buffer(p0)->sw_if_index[VLIB_RX]);
2016
2017           mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
2018
2019           leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
2020
2021           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 0);
2022
2023           /* Treat IP frag packets as "experimental" protocol for now
2024              until support of IP frag reassembly is implemented */
2025           proto0 = ip4_is_fragment(ip0) ? 0xfe : ip0->protocol;
2026           is_udp0 = proto0 == IP_PROTOCOL_UDP;
2027           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
2028
2029           flags0 = p0->flags;
2030
2031           good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
2032
2033           udp0 = ip4_next_header (ip0);
2034
2035           /* Don't verify UDP checksum for packets with explicit zero checksum. */
2036           good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
2037
2038           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
2039
2040           /* Verify UDP length. */
2041           ip_len0 = clib_net_to_host_u16 (ip0->length);
2042           udp_len0 = clib_net_to_host_u16 (udp0->length);
2043
2044           len_diff0 = ip_len0 - udp_len0;
2045
2046           len_diff0 = is_udp0 ? len_diff0 : 0;
2047
2048           if (PREDICT_FALSE (! (is_tcp_udp0 & good_tcp_udp0)))
2049             {
2050               if (is_tcp_udp0)
2051                 {
2052                   if (is_tcp_udp0
2053                       && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
2054                     flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
2055                   good_tcp_udp0 =
2056                     (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
2057                   good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
2058                 }
2059             }
2060
2061           good_tcp_udp0 &= len_diff0 >= 0;
2062
2063           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
2064
2065           error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
2066
2067           error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
2068
2069           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
2070           error0 = (is_tcp_udp0 && ! good_tcp_udp0
2071                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0
2072                     : error0);
2073
2074           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
2075
2076           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2077           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
2078
2079           ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
2080                                                            &ip0->src_address,
2081                                                            /* no_default_route */ 1));
2082
2083           adj0 = ip_get_adjacency (lm, adj_index0);
2084
2085           /* Must have a route to source otherwise we drop the packet. */
2086           error0 = (error0 == IP4_ERROR_UNKNOWN_PROTOCOL
2087                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_REWRITE
2088                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP
2089                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_LOCAL
2090                     && ip0->dst_address.as_u32 != 0xFFFFFFFF
2091                     ? IP4_ERROR_SRC_LOOKUP_MISS
2092                     : error0);
2093
2094           next0 = lm->local_next_by_ip_protocol[proto0];
2095
2096           next0 = error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
2097
2098           p0->error = error0? error_node->errors[error0] : 0;
2099
2100           if (PREDICT_FALSE (next0 != next_index))
2101             {
2102               n_left_to_next += 1;
2103               vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2104
2105               next_index = next0;
2106               vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2107               to_next[0] = pi0;
2108               to_next += 1;
2109               n_left_to_next -= 1;
2110             }
2111         }
2112
2113       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2114     }
2115
2116   return frame->n_vectors;
2117 }
2118
2119 VLIB_REGISTER_NODE (ip4_local_node,static) = {
2120   .function = ip4_local,
2121   .name = "ip4-local",
2122   .vector_size = sizeof (u32),
2123
2124   .format_trace = format_ip4_forward_next_trace,
2125
2126   .n_next_nodes = IP_LOCAL_N_NEXT,
2127   .next_nodes = {
2128     [IP_LOCAL_NEXT_DROP] = "error-drop",
2129     [IP_LOCAL_NEXT_PUNT] = "error-punt",
2130     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
2131     [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",
2132   },
2133 };
2134
2135 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local)
2136
2137 void ip4_register_protocol (u32 protocol, u32 node_index)
2138 {
2139   vlib_main_t * vm = vlib_get_main();
2140   ip4_main_t * im = &ip4_main;
2141   ip_lookup_main_t * lm = &im->lookup_main;
2142
2143   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
2144   lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip4_local_node.index, node_index);
2145 }
2146
2147 static clib_error_t *
2148 show_ip_local_command_fn (vlib_main_t * vm,
2149                           unformat_input_t * input,
2150                          vlib_cli_command_t * cmd)
2151 {
2152   ip4_main_t * im = &ip4_main;
2153   ip_lookup_main_t * lm = &im->lookup_main;
2154   int i;
2155
2156   vlib_cli_output (vm, "Protocols handled by ip4_local");
2157   for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
2158     {
2159       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
2160         vlib_cli_output (vm, "%d", i);
2161     }
2162   return 0;
2163 }
2164
2165
2166
2167 VLIB_CLI_COMMAND (show_ip_local, static) = {
2168   .path = "show ip local",
2169   .function = show_ip_local_command_fn,
2170   .short_help = "Show ip local protocol table",
2171 };
2172
2173 static uword
2174 ip4_arp (vlib_main_t * vm,
2175          vlib_node_runtime_t * node,
2176          vlib_frame_t * frame)
2177 {
2178   vnet_main_t * vnm = vnet_get_main();
2179   ip4_main_t * im = &ip4_main;
2180   ip_lookup_main_t * lm = &im->lookup_main;
2181   u32 * from, * to_next_drop;
2182   uword n_left_from, n_left_to_next_drop, next_index;
2183   static f64 time_last_seed_change = -1e100;
2184   static u32 hash_seeds[3];
2185   static uword hash_bitmap[256 / BITS (uword)];
2186   f64 time_now;
2187
2188   if (node->flags & VLIB_NODE_FLAG_TRACE)
2189     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2190
2191   time_now = vlib_time_now (vm);
2192   if (time_now - time_last_seed_change > 1e-3)
2193     {
2194       uword i;
2195       u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
2196                                              sizeof (hash_seeds));
2197       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
2198         hash_seeds[i] = r[i];
2199
2200       /* Mark all hash keys as been no-seen before. */
2201       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
2202         hash_bitmap[i] = 0;
2203
2204       time_last_seed_change = time_now;
2205     }
2206
2207   from = vlib_frame_vector_args (frame);
2208   n_left_from = frame->n_vectors;
2209   next_index = node->cached_next_index;
2210   if (next_index == IP4_ARP_NEXT_DROP)
2211     next_index = IP4_ARP_N_NEXT; /* point to first interface */
2212
2213   while (n_left_from > 0)
2214     {
2215       vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
2216                            to_next_drop, n_left_to_next_drop);
2217
2218       while (n_left_from > 0 && n_left_to_next_drop > 0)
2219         {
2220           vlib_buffer_t * p0;
2221           ip4_header_t * ip0;
2222           ethernet_header_t * eh0;
2223           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
2224           uword bm0;
2225           ip_adjacency_t * adj0;
2226
2227           pi0 = from[0];
2228
2229           p0 = vlib_get_buffer (vm, pi0);
2230
2231           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2232           adj0 = ip_get_adjacency (lm, adj_index0);
2233           ip0 = vlib_buffer_get_current (p0);
2234
2235           /* If packet destination is not local, send ARP to next hop */
2236           if (adj0->arp.next_hop.ip4.as_u32)
2237             ip0->dst_address.data_u32 = adj0->arp.next_hop.ip4.as_u32;
2238
2239           /*
2240            * if ip4_rewrite_local applied the IP_LOOKUP_NEXT_ARP
2241            * rewrite to this packet, we need to skip it here.
2242            * Note, to distinguish from src IP addr *.8.6.*, we
2243            * check for a bcast eth dest instead of IPv4 version.
2244            */
2245           eh0 = (ethernet_header_t*)ip0;
2246           if ((ip0->ip_version_and_header_length & 0xF0) != 0x40)
2247             {
2248               u32 vlan_num = 0;
2249               u16 * etype = &eh0->type;
2250               while ((*etype == clib_host_to_net_u16 (0x8100)) //dot1q
2251                   || (*etype == clib_host_to_net_u16 (0x88a8)))//dot1ad
2252                 {
2253                   vlan_num += 1;
2254                   etype += 2; //vlan tag also 16 bits, same as etype
2255                 }
2256               if (*etype == clib_host_to_net_u16 (0x0806))     //arp
2257                 {
2258                   vlib_buffer_advance (
2259                       p0, sizeof(ethernet_header_t) + (4*vlan_num));
2260                   ip0 = vlib_buffer_get_current (p0);
2261                 }
2262             }
2263
2264           a0 = hash_seeds[0];
2265           b0 = hash_seeds[1];
2266           c0 = hash_seeds[2];
2267
2268           sw_if_index0 = adj0->rewrite_header.sw_if_index;
2269           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
2270
2271           a0 ^= ip0->dst_address.data_u32;
2272           b0 ^= sw_if_index0;
2273
2274           hash_v3_finalize32 (a0, b0, c0);
2275
2276           c0 &= BITS (hash_bitmap) - 1;
2277           c0 = c0 / BITS (uword);
2278           m0 = (uword) 1 << (c0 % BITS (uword));
2279
2280           bm0 = hash_bitmap[c0];
2281           drop0 = (bm0 & m0) != 0;
2282
2283           /* Mark it as seen. */
2284           hash_bitmap[c0] = bm0 | m0;
2285
2286           from += 1;
2287           n_left_from -= 1;
2288           to_next_drop[0] = pi0;
2289           to_next_drop += 1;
2290           n_left_to_next_drop -= 1;
2291
2292           p0->error = node->errors[drop0 ? IP4_ARP_ERROR_DROP : IP4_ARP_ERROR_REQUEST_SENT];
2293
2294           if (drop0)
2295             continue;
2296
2297           /*
2298            * Can happen if the control-plane is programming tables
2299            * with traffic flowing; at least that's today's lame excuse.
2300            */
2301           if (adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP)
2302             {
2303               p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
2304             }
2305           else
2306           /* Send ARP request. */
2307           {
2308             u32 bi0 = 0;
2309             vlib_buffer_t * b0;
2310             ethernet_arp_header_t * h0;
2311             vnet_hw_interface_t * hw_if0;
2312
2313             h0 = vlib_packet_template_get_packet (vm, &im->ip4_arp_request_packet_template, &bi0);
2314
2315             /* Add rewrite/encap string for ARP packet. */
2316             vnet_rewrite_one_header (adj0[0], h0, sizeof (ethernet_header_t));
2317
2318             hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2319
2320             /* Src ethernet address in ARP header. */
2321             clib_memcpy (h0->ip4_over_ethernet[0].ethernet, hw_if0->hw_address,
2322                     sizeof (h0->ip4_over_ethernet[0].ethernet));
2323
2324             ip4_src_address_for_packet (im, p0, &h0->ip4_over_ethernet[0].ip4, sw_if_index0);
2325
2326             /* Copy in destination address we are requesting. */
2327             h0->ip4_over_ethernet[1].ip4.data_u32 = ip0->dst_address.data_u32;
2328
2329             vlib_buffer_copy_trace_flag (vm, p0, bi0);
2330             b0 = vlib_get_buffer (vm, bi0);
2331             vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
2332
2333             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2334
2335             vlib_set_next_frame_buffer (vm, node, adj0->rewrite_header.next_index, bi0);
2336           }
2337         }
2338
2339       vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
2340     }
2341
2342   return frame->n_vectors;
2343 }
2344
2345 static char * ip4_arp_error_strings[] = {
2346   [IP4_ARP_ERROR_DROP] = "address overflow drops",
2347   [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
2348   [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
2349   [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
2350   [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
2351 };
2352
2353 VLIB_REGISTER_NODE (ip4_arp_node) = {
2354   .function = ip4_arp,
2355   .name = "ip4-arp",
2356   .vector_size = sizeof (u32),
2357
2358   .format_trace = format_ip4_forward_next_trace,
2359
2360   .n_errors = ARRAY_LEN (ip4_arp_error_strings),
2361   .error_strings = ip4_arp_error_strings,
2362
2363   .n_next_nodes = IP4_ARP_N_NEXT,
2364   .next_nodes = {
2365     [IP4_ARP_NEXT_DROP] = "error-drop",
2366   },
2367 };
2368
2369 #define foreach_notrace_ip4_arp_error           \
2370 _(DROP)                                         \
2371 _(REQUEST_SENT)                                 \
2372 _(REPLICATE_DROP)                               \
2373 _(REPLICATE_FAIL)
2374
2375 clib_error_t * arp_notrace_init (vlib_main_t * vm)
2376 {
2377   vlib_node_runtime_t *rt =
2378     vlib_node_get_runtime (vm, ip4_arp_node.index);
2379
2380   /* don't trace ARP request packets */
2381 #define _(a)                                    \
2382     vnet_pcap_drop_trace_filter_add_del         \
2383         (rt->errors[IP4_ARP_ERROR_##a],         \
2384          1 /* is_add */);
2385     foreach_notrace_ip4_arp_error;
2386 #undef _
2387   return 0;
2388 }
2389
2390 VLIB_INIT_FUNCTION(arp_notrace_init);
2391
2392
2393 /* Send an ARP request to see if given destination is reachable on given interface. */
2394 clib_error_t *
2395 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
2396 {
2397   vnet_main_t * vnm = vnet_get_main();
2398   ip4_main_t * im = &ip4_main;
2399   ethernet_arp_header_t * h;
2400   ip4_address_t * src;
2401   ip_interface_address_t * ia;
2402   ip_adjacency_t * adj;
2403   vnet_hw_interface_t * hi;
2404   vnet_sw_interface_t * si;
2405   vlib_buffer_t * b;
2406   u32 bi = 0;
2407
2408   si = vnet_get_sw_interface (vnm, sw_if_index);
2409
2410   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2411     {
2412       return clib_error_return (0, "%U: interface %U down",
2413                                 format_ip4_address, dst,
2414                                 format_vnet_sw_if_index_name, vnm,
2415                                 sw_if_index);
2416     }
2417
2418   src = ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2419   if (! src)
2420     {
2421       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2422       return clib_error_return
2423         (0, "no matching interface address for destination %U (interface %U)",
2424          format_ip4_address, dst,
2425          format_vnet_sw_if_index_name, vnm, sw_if_index);
2426     }
2427
2428   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
2429
2430   h = vlib_packet_template_get_packet (vm, &im->ip4_arp_request_packet_template, &bi);
2431
2432   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2433
2434   clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address, sizeof (h->ip4_over_ethernet[0].ethernet));
2435
2436   h->ip4_over_ethernet[0].ip4 = src[0];
2437   h->ip4_over_ethernet[1].ip4 = dst[0];
2438
2439   b = vlib_get_buffer (vm, bi);
2440   vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2441
2442   /* Add encapsulation string for software interface (e.g. ethernet header). */
2443   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2444   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2445
2446   {
2447     vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
2448     u32 * to_next = vlib_frame_vector_args (f);
2449     to_next[0] = bi;
2450     f->n_vectors = 1;
2451     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2452   }
2453
2454   return /* no error */ 0;
2455 }
2456
2457 typedef enum {
2458   IP4_REWRITE_NEXT_DROP,
2459   IP4_REWRITE_NEXT_ARP,
2460 } ip4_rewrite_next_t;
2461
2462 always_inline uword
2463 ip4_rewrite_inline (vlib_main_t * vm,
2464                     vlib_node_runtime_t * node,
2465                     vlib_frame_t * frame,
2466                     int rewrite_for_locally_received_packets)
2467 {
2468   ip_lookup_main_t * lm = &ip4_main.lookup_main;
2469   u32 * from = vlib_frame_vector_args (frame);
2470   u32 n_left_from, n_left_to_next, * to_next, next_index;
2471   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip4_input_node.index);
2472   vlib_rx_or_tx_t adj_rx_tx = rewrite_for_locally_received_packets ? VLIB_RX : VLIB_TX;
2473
2474   n_left_from = frame->n_vectors;
2475   next_index = node->cached_next_index;
2476   u32 cpu_index = os_get_cpu_number();
2477
2478   while (n_left_from > 0)
2479     {
2480       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2481
2482       while (n_left_from >= 4 && n_left_to_next >= 2)
2483         {
2484           ip_adjacency_t * adj0, * adj1;
2485           vlib_buffer_t * p0, * p1;
2486           ip4_header_t * ip0, * ip1;
2487           u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
2488           u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
2489           u32 next0_override, next1_override;
2490
2491           if (rewrite_for_locally_received_packets)
2492               next0_override = next1_override = 0;
2493
2494           /* Prefetch next iteration. */
2495           {
2496             vlib_buffer_t * p2, * p3;
2497
2498             p2 = vlib_get_buffer (vm, from[2]);
2499             p3 = vlib_get_buffer (vm, from[3]);
2500
2501             vlib_prefetch_buffer_header (p2, STORE);
2502             vlib_prefetch_buffer_header (p3, STORE);
2503
2504             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2505             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2506           }
2507
2508           pi0 = to_next[0] = from[0];
2509           pi1 = to_next[1] = from[1];
2510
2511           from += 2;
2512           n_left_from -= 2;
2513           to_next += 2;
2514           n_left_to_next -= 2;
2515
2516           p0 = vlib_get_buffer (vm, pi0);
2517           p1 = vlib_get_buffer (vm, pi1);
2518
2519           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2520           adj_index1 = vnet_buffer (p1)->ip.adj_index[adj_rx_tx];
2521
2522           /* We should never rewrite a pkt using the MISS adjacency */
2523           ASSERT(adj_index0 && adj_index1);
2524
2525           ip0 = vlib_buffer_get_current (p0);
2526           ip1 = vlib_buffer_get_current (p1);
2527
2528           error0 = error1 = IP4_ERROR_NONE;
2529
2530           /* Decrement TTL & update checksum.
2531              Works either endian, so no need for byte swap. */
2532           if (! rewrite_for_locally_received_packets)
2533             {
2534               i32 ttl0 = ip0->ttl, ttl1 = ip1->ttl;
2535
2536               /* Input node should have reject packets with ttl 0. */
2537               ASSERT (ip0->ttl > 0);
2538               ASSERT (ip1->ttl > 0);
2539
2540               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2541               checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2542
2543               checksum0 += checksum0 >= 0xffff;
2544               checksum1 += checksum1 >= 0xffff;
2545
2546               ip0->checksum = checksum0;
2547               ip1->checksum = checksum1;
2548
2549               ttl0 -= 1;
2550               ttl1 -= 1;
2551
2552               ip0->ttl = ttl0;
2553               ip1->ttl = ttl1;
2554
2555               error0 = ttl0 <= 0 ? IP4_ERROR_TIME_EXPIRED : error0;
2556               error1 = ttl1 <= 0 ? IP4_ERROR_TIME_EXPIRED : error1;
2557
2558               /* Verify checksum. */
2559               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2560               ASSERT (ip1->checksum == ip4_header_checksum (ip1));
2561             }
2562
2563           /* Rewrite packet header and updates lengths. */
2564           adj0 = ip_get_adjacency (lm, adj_index0);
2565           adj1 = ip_get_adjacency (lm, adj_index1);
2566
2567           if (rewrite_for_locally_received_packets)
2568             {
2569               /*
2570                * If someone sends e.g. an icmp4 w/ src = dst = interface addr,
2571                * we end up here with a local adjacency in hand
2572                * The local adj rewrite data is 0xfefe on purpose.
2573                * Bad engineer, no donut for you.
2574                */
2575               if (PREDICT_FALSE(adj0->lookup_next_index
2576                                 == IP_LOOKUP_NEXT_LOCAL))
2577                 error0 = IP4_ERROR_SPOOFED_LOCAL_PACKETS;
2578               if (PREDICT_FALSE(adj0->lookup_next_index
2579                                 == IP_LOOKUP_NEXT_ARP))
2580                 next0_override = IP4_REWRITE_NEXT_ARP;
2581               if (PREDICT_FALSE(adj1->lookup_next_index
2582                                 == IP_LOOKUP_NEXT_LOCAL))
2583                 error1 = IP4_ERROR_SPOOFED_LOCAL_PACKETS;
2584               if (PREDICT_FALSE(adj1->lookup_next_index
2585                                 == IP_LOOKUP_NEXT_ARP))
2586                 next1_override = IP4_REWRITE_NEXT_ARP;
2587             }
2588
2589           /* Worth pipelining. No guarantee that adj0,1 are hot... */
2590           rw_len0 = adj0[0].rewrite_header.data_bytes;
2591           rw_len1 = adj1[0].rewrite_header.data_bytes;
2592           next0 = (error0 == IP4_ERROR_NONE)
2593             ? adj0[0].rewrite_header.next_index : 0;
2594
2595           if (rewrite_for_locally_received_packets)
2596               next0 = next0 && next0_override ? next0_override : next0;
2597
2598           next1 = (error1 == IP4_ERROR_NONE)
2599             ? adj1[0].rewrite_header.next_index : 0;
2600
2601           if (rewrite_for_locally_received_packets)
2602               next1 = next1 && next1_override ? next1_override : next1;
2603
2604           /*
2605            * We've already accounted for an ethernet_header_t elsewhere
2606            */
2607           if (PREDICT_FALSE (rw_len0 > sizeof(ethernet_header_t)))
2608               vlib_increment_combined_counter
2609                   (&lm->adjacency_counters,
2610                    cpu_index, adj_index0,
2611                    /* packet increment */ 0,
2612                    /* byte increment */ rw_len0-sizeof(ethernet_header_t));
2613
2614           if (PREDICT_FALSE (rw_len1 > sizeof(ethernet_header_t)))
2615               vlib_increment_combined_counter
2616                   (&lm->adjacency_counters,
2617                    cpu_index, adj_index1,
2618                    /* packet increment */ 0,
2619                    /* byte increment */ rw_len1-sizeof(ethernet_header_t));
2620
2621           /* Check MTU of outgoing interface. */
2622           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2623                     ? IP4_ERROR_MTU_EXCEEDED
2624                     : error0);
2625           error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
2626                     ? IP4_ERROR_MTU_EXCEEDED
2627                     : error1);
2628
2629           p0->current_data -= rw_len0;
2630           p1->current_data -= rw_len1;
2631
2632           p0->current_length += rw_len0;
2633           p1->current_length += rw_len1;
2634
2635           vnet_buffer (p0)->sw_if_index[VLIB_TX] = adj0[0].rewrite_header.sw_if_index;
2636           vnet_buffer (p1)->sw_if_index[VLIB_TX] = adj1[0].rewrite_header.sw_if_index;
2637
2638           p0->error = error_node->errors[error0];
2639           p1->error = error_node->errors[error1];
2640
2641           /* Guess we are only writing on simple Ethernet header. */
2642           vnet_rewrite_two_headers (adj0[0], adj1[0],
2643                                     ip0, ip1,
2644                                     sizeof (ethernet_header_t));
2645
2646           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2647                                            to_next, n_left_to_next,
2648                                            pi0, pi1, next0, next1);
2649         }
2650
2651       while (n_left_from > 0 && n_left_to_next > 0)
2652         {
2653           ip_adjacency_t * adj0;
2654           vlib_buffer_t * p0;
2655           ip4_header_t * ip0;
2656           u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2657           u32 next0_override;
2658
2659           if (rewrite_for_locally_received_packets)
2660               next0_override = 0;
2661
2662           pi0 = to_next[0] = from[0];
2663
2664           p0 = vlib_get_buffer (vm, pi0);
2665
2666           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2667
2668           /* We should never rewrite a pkt using the MISS adjacency */
2669           ASSERT(adj_index0);
2670
2671           adj0 = ip_get_adjacency (lm, adj_index0);
2672
2673           ip0 = vlib_buffer_get_current (p0);
2674
2675           error0 = IP4_ERROR_NONE;
2676           next0 = 0;            /* drop on error */
2677
2678           /* Decrement TTL & update checksum. */
2679           if (! rewrite_for_locally_received_packets)
2680             {
2681               i32 ttl0 = ip0->ttl;
2682
2683               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2684
2685               checksum0 += checksum0 >= 0xffff;
2686
2687               ip0->checksum = checksum0;
2688
2689               ASSERT (ip0->ttl > 0);
2690
2691               ttl0 -= 1;
2692
2693               ip0->ttl = ttl0;
2694
2695               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2696
2697               error0 = ttl0 <= 0 ? IP4_ERROR_TIME_EXPIRED : error0;
2698             }
2699
2700           if (rewrite_for_locally_received_packets)
2701             {
2702               /*
2703                * If someone sends e.g. an icmp4 w/ src = dst = interface addr,
2704                * we end up here with a local adjacency in hand
2705                * The local adj rewrite data is 0xfefe on purpose.
2706                * Bad engineer, no donut for you.
2707                */
2708               if (PREDICT_FALSE(adj0->lookup_next_index
2709                                 == IP_LOOKUP_NEXT_LOCAL))
2710                 error0 = IP4_ERROR_SPOOFED_LOCAL_PACKETS;
2711               /*
2712                * We have to override the next_index in ARP adjacencies,
2713                * because they're set up for ip4-arp, not this node...
2714                */
2715               if (PREDICT_FALSE(adj0->lookup_next_index
2716                                 == IP_LOOKUP_NEXT_ARP))
2717                 next0_override = IP4_REWRITE_NEXT_ARP;
2718             }
2719
2720           /* Guess we are only writing on simple Ethernet header. */
2721           vnet_rewrite_one_header (adj0[0], ip0,
2722                                    sizeof (ethernet_header_t));
2723
2724           /* Update packet buffer attributes/set output interface. */
2725           rw_len0 = adj0[0].rewrite_header.data_bytes;
2726
2727           if (PREDICT_FALSE (rw_len0 > sizeof(ethernet_header_t)))
2728               vlib_increment_combined_counter
2729                   (&lm->adjacency_counters,
2730                    cpu_index, adj_index0,
2731                    /* packet increment */ 0,
2732                    /* byte increment */ rw_len0-sizeof(ethernet_header_t));
2733
2734           /* Check MTU of outgoing interface. */
2735           error0 = (vlib_buffer_length_in_chain (vm, p0)
2736                     > adj0[0].rewrite_header.max_l3_packet_bytes
2737                     ? IP4_ERROR_MTU_EXCEEDED
2738                     : error0);
2739
2740           p0->error = error_node->errors[error0];
2741           p0->current_data -= rw_len0;
2742           p0->current_length += rw_len0;
2743           vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2744             adj0[0].rewrite_header.sw_if_index;
2745
2746           next0 = (error0 == IP4_ERROR_NONE)
2747             ? adj0[0].rewrite_header.next_index : 0;
2748
2749           if (rewrite_for_locally_received_packets)
2750               next0 = next0 && next0_override ? next0_override : next0;
2751
2752           from += 1;
2753           n_left_from -= 1;
2754           to_next += 1;
2755           n_left_to_next -= 1;
2756
2757           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2758                                            to_next, n_left_to_next,
2759                                            pi0, next0);
2760         }
2761
2762       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2763     }
2764
2765   /* Need to do trace after rewrites to pick up new packet data. */
2766   if (node->flags & VLIB_NODE_FLAG_TRACE)
2767     ip4_forward_next_trace (vm, node, frame, adj_rx_tx);
2768
2769   return frame->n_vectors;
2770 }
2771
2772 static uword
2773 ip4_rewrite_transit (vlib_main_t * vm,
2774                      vlib_node_runtime_t * node,
2775                      vlib_frame_t * frame)
2776 {
2777   return ip4_rewrite_inline (vm, node, frame,
2778                              /* rewrite_for_locally_received_packets */ 0);
2779 }
2780
2781 static uword
2782 ip4_rewrite_local (vlib_main_t * vm,
2783                    vlib_node_runtime_t * node,
2784                    vlib_frame_t * frame)
2785 {
2786   return ip4_rewrite_inline (vm, node, frame,
2787                              /* rewrite_for_locally_received_packets */ 1);
2788 }
2789
2790 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2791   .function = ip4_rewrite_transit,
2792   .name = "ip4-rewrite-transit",
2793   .vector_size = sizeof (u32),
2794
2795   .format_trace = format_ip4_rewrite_trace,
2796
2797   .n_next_nodes = 2,
2798   .next_nodes = {
2799     [IP4_REWRITE_NEXT_DROP] = "error-drop",
2800     [IP4_REWRITE_NEXT_ARP] = "ip4-arp",
2801   },
2802 };
2803
2804 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite_transit)
2805
2806 VLIB_REGISTER_NODE (ip4_rewrite_local_node) = {
2807   .function = ip4_rewrite_local,
2808   .name = "ip4-rewrite-local",
2809   .vector_size = sizeof (u32),
2810
2811   .sibling_of = "ip4-rewrite-transit",
2812
2813   .format_trace = format_ip4_rewrite_trace,
2814
2815   .n_next_nodes = 0,
2816 };
2817
2818 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_local_node, ip4_rewrite_local)
2819
2820 static clib_error_t *
2821 add_del_interface_table (vlib_main_t * vm,
2822                          unformat_input_t * input,
2823                          vlib_cli_command_t * cmd)
2824 {
2825   vnet_main_t * vnm = vnet_get_main();
2826   clib_error_t * error = 0;
2827   u32 sw_if_index, table_id;
2828
2829   sw_if_index = ~0;
2830
2831   if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2832     {
2833       error = clib_error_return (0, "unknown interface `%U'",
2834                                  format_unformat_error, input);
2835       goto done;
2836     }
2837
2838   if (unformat (input, "%d", &table_id))
2839     ;
2840   else
2841     {
2842       error = clib_error_return (0, "expected table id `%U'",
2843                                  format_unformat_error, input);
2844       goto done;
2845     }
2846
2847   {
2848     ip4_main_t * im = &ip4_main;
2849     ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_id, IP4_ROUTE_FLAG_TABLE_ID);
2850
2851     if (fib)
2852       {
2853         vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
2854         im->fib_index_by_sw_if_index[sw_if_index] = fib->index;
2855     }
2856   }
2857
2858  done:
2859   return error;
2860 }
2861
2862 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) = {
2863   .path = "set interface ip table",
2864   .function = add_del_interface_table,
2865   .short_help = "Add/delete FIB table id for interface",
2866 };
2867
2868
2869 static uword
2870 ip4_lookup_multicast (vlib_main_t * vm,
2871                       vlib_node_runtime_t * node,
2872                       vlib_frame_t * frame)
2873 {
2874   ip4_main_t * im = &ip4_main;
2875   ip_lookup_main_t * lm = &im->lookup_main;
2876   vlib_combined_counter_main_t * cm = &im->lookup_main.adjacency_counters;
2877   u32 n_left_from, n_left_to_next, * from, * to_next;
2878   ip_lookup_next_t next;
2879   u32 cpu_index = os_get_cpu_number();
2880
2881   from = vlib_frame_vector_args (frame);
2882   n_left_from = frame->n_vectors;
2883   next = node->cached_next_index;
2884
2885   while (n_left_from > 0)
2886     {
2887       vlib_get_next_frame (vm, node, next,
2888                            to_next, n_left_to_next);
2889
2890       while (n_left_from >= 4 && n_left_to_next >= 2)
2891         {
2892           vlib_buffer_t * p0, * p1;
2893           u32 pi0, pi1, adj_index0, adj_index1, wrong_next;
2894           ip_lookup_next_t next0, next1;
2895           ip4_header_t * ip0, * ip1;
2896           ip_adjacency_t * adj0, * adj1;
2897           u32 fib_index0, fib_index1;
2898           u32 flow_hash_config0, flow_hash_config1;
2899
2900           /* Prefetch next iteration. */
2901           {
2902             vlib_buffer_t * p2, * p3;
2903
2904             p2 = vlib_get_buffer (vm, from[2]);
2905             p3 = vlib_get_buffer (vm, from[3]);
2906
2907             vlib_prefetch_buffer_header (p2, LOAD);
2908             vlib_prefetch_buffer_header (p3, LOAD);
2909
2910             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
2911             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
2912           }
2913
2914           pi0 = to_next[0] = from[0];
2915           pi1 = to_next[1] = from[1];
2916
2917           p0 = vlib_get_buffer (vm, pi0);
2918           p1 = vlib_get_buffer (vm, pi1);
2919
2920           ip0 = vlib_buffer_get_current (p0);
2921           ip1 = vlib_buffer_get_current (p1);
2922
2923           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
2924           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
2925           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
2926             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
2927           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
2928             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
2929
2930           adj_index0 = ip4_fib_lookup_buffer (im, fib_index0,
2931                                               &ip0->dst_address, p0);
2932           adj_index1 = ip4_fib_lookup_buffer (im, fib_index1,
2933                                               &ip1->dst_address, p1);
2934
2935           adj0 = ip_get_adjacency (lm, adj_index0);
2936           adj1 = ip_get_adjacency (lm, adj_index1);
2937
2938           next0 = adj0->lookup_next_index;
2939           next1 = adj1->lookup_next_index;
2940
2941           flow_hash_config0 =
2942               vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
2943
2944           flow_hash_config1 =
2945               vec_elt_at_index (im->fibs, fib_index1)->flow_hash_config;
2946
2947           vnet_buffer (p0)->ip.flow_hash = ip4_compute_flow_hash
2948               (ip0, flow_hash_config0);
2949
2950           vnet_buffer (p1)->ip.flow_hash = ip4_compute_flow_hash
2951               (ip1, flow_hash_config1);
2952
2953           ASSERT (adj0->n_adj > 0);
2954           ASSERT (adj1->n_adj > 0);
2955           ASSERT (is_pow2 (adj0->n_adj));
2956           ASSERT (is_pow2 (adj1->n_adj));
2957           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
2958           adj_index1 += (vnet_buffer (p1)->ip.flow_hash & (adj1->n_adj - 1));
2959
2960           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
2961           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
2962
2963           if (1) /* $$$$$$ HACK FIXME */
2964           vlib_increment_combined_counter
2965               (cm, cpu_index, adj_index0, 1,
2966                vlib_buffer_length_in_chain (vm, p0));
2967           if (1) /* $$$$$$ HACK FIXME */
2968           vlib_increment_combined_counter
2969               (cm, cpu_index, adj_index1, 1,
2970                vlib_buffer_length_in_chain (vm, p1));
2971
2972           from += 2;
2973           to_next += 2;
2974           n_left_to_next -= 2;
2975           n_left_from -= 2;
2976
2977           wrong_next = (next0 != next) + 2*(next1 != next);
2978           if (PREDICT_FALSE (wrong_next != 0))
2979             {
2980               switch (wrong_next)
2981                 {
2982                 case 1:
2983                   /* A B A */
2984                   to_next[-2] = pi1;
2985                   to_next -= 1;
2986                   n_left_to_next += 1;
2987                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
2988                   break;
2989
2990                 case 2:
2991                   /* A A B */
2992                   to_next -= 1;
2993                   n_left_to_next += 1;
2994                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
2995                   break;
2996
2997                 case 3:
2998                   /* A B C */
2999                   to_next -= 2;
3000                   n_left_to_next += 2;
3001                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
3002                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
3003                   if (next0 == next1)
3004                     {
3005                       /* A B B */
3006                       vlib_put_next_frame (vm, node, next, n_left_to_next);
3007                       next = next1;
3008                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
3009                     }
3010                 }
3011             }
3012         }
3013
3014       while (n_left_from > 0 && n_left_to_next > 0)
3015         {
3016           vlib_buffer_t * p0;
3017           ip4_header_t * ip0;
3018           u32 pi0, adj_index0;
3019           ip_lookup_next_t next0;
3020           ip_adjacency_t * adj0;
3021           u32 fib_index0;
3022           u32 flow_hash_config0;
3023
3024           pi0 = from[0];
3025           to_next[0] = pi0;
3026
3027           p0 = vlib_get_buffer (vm, pi0);
3028
3029           ip0 = vlib_buffer_get_current (p0);
3030
3031           fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
3032                                 vnet_buffer (p0)->sw_if_index[VLIB_RX]);
3033           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
3034               fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
3035
3036           adj_index0 = ip4_fib_lookup_buffer (im, fib_index0,
3037                                               &ip0->dst_address, p0);
3038
3039           adj0 = ip_get_adjacency (lm, adj_index0);
3040
3041           next0 = adj0->lookup_next_index;
3042
3043           flow_hash_config0 =
3044               vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
3045
3046           vnet_buffer (p0)->ip.flow_hash =
3047             ip4_compute_flow_hash (ip0, flow_hash_config0);
3048
3049           ASSERT (adj0->n_adj > 0);
3050           ASSERT (is_pow2 (adj0->n_adj));
3051           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
3052
3053           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
3054
3055           if (1) /* $$$$$$ HACK FIXME */
3056               vlib_increment_combined_counter
3057                   (cm, cpu_index, adj_index0, 1,
3058                    vlib_buffer_length_in_chain (vm, p0));
3059
3060           from += 1;
3061           to_next += 1;
3062           n_left_to_next -= 1;
3063           n_left_from -= 1;
3064
3065           if (PREDICT_FALSE (next0 != next))
3066             {
3067               n_left_to_next += 1;
3068               vlib_put_next_frame (vm, node, next, n_left_to_next);
3069               next = next0;
3070               vlib_get_next_frame (vm, node, next,
3071                                    to_next, n_left_to_next);
3072               to_next[0] = pi0;
3073               to_next += 1;
3074               n_left_to_next -= 1;
3075             }
3076         }
3077
3078       vlib_put_next_frame (vm, node, next, n_left_to_next);
3079     }
3080
3081   if (node->flags & VLIB_NODE_FLAG_TRACE)
3082       ip4_forward_next_trace(vm, node, frame, VLIB_TX);
3083
3084   return frame->n_vectors;
3085 }
3086
3087 VLIB_REGISTER_NODE (ip4_lookup_multicast_node,static) = {
3088   .function = ip4_lookup_multicast,
3089   .name = "ip4-lookup-multicast",
3090   .vector_size = sizeof (u32),
3091
3092   .format_trace = format_ip4_lookup_trace,
3093
3094   .n_next_nodes = IP_LOOKUP_N_NEXT,
3095   .next_nodes = IP4_LOOKUP_NEXT_NODES,
3096 };
3097
3098 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_multicast_node, ip4_lookup_multicast)
3099
3100 VLIB_REGISTER_NODE (ip4_multicast_node,static) = {
3101   .function = ip4_drop,
3102   .name = "ip4-multicast",
3103   .vector_size = sizeof (u32),
3104
3105   .format_trace = format_ip4_forward_next_trace,
3106
3107   .n_next_nodes = 1,
3108   .next_nodes = {
3109     [0] = "error-drop",
3110   },
3111 };
3112
3113 int ip4_lookup_validate (ip4_address_t *a, u32 fib_index0)
3114 {
3115   ip4_main_t * im = &ip4_main;
3116   ip4_fib_mtrie_t * mtrie0;
3117   ip4_fib_mtrie_leaf_t leaf0;
3118   u32 adj_index0;
3119
3120   mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
3121
3122   leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
3123   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 0);
3124   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 1);
3125   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
3126   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
3127
3128   /* Handle default route. */
3129   leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
3130
3131   adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
3132
3133   return adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
3134                                                   a,
3135                                                   /* no_default_route */ 0);
3136 }
3137
3138 static clib_error_t *
3139 test_lookup_command_fn (vlib_main_t * vm,
3140                         unformat_input_t * input,
3141                         vlib_cli_command_t * cmd)
3142 {
3143   u32 table_id = 0;
3144   f64 count = 1;
3145   u32 n;
3146   int i;
3147   ip4_address_t ip4_base_address;
3148   u64 errors = 0;
3149
3150   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3151       if (unformat (input, "table %d", &table_id))
3152         ;
3153       else if (unformat (input, "count %f", &count))
3154         ;
3155
3156       else if (unformat (input, "%U",
3157                          unformat_ip4_address, &ip4_base_address))
3158         ;
3159       else
3160         return clib_error_return (0, "unknown input `%U'",
3161                                   format_unformat_error, input);
3162   }
3163
3164   n = count;
3165
3166   for (i = 0; i < n; i++)
3167     {
3168       if (!ip4_lookup_validate (&ip4_base_address, table_id))
3169         errors++;
3170
3171       ip4_base_address.as_u32 =
3172         clib_host_to_net_u32 (1 +
3173                               clib_net_to_host_u32 (ip4_base_address.as_u32));
3174     }
3175
3176   if (errors)
3177     vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
3178   else
3179     vlib_cli_output (vm, "No errors in %d lookups\n", n);
3180
3181   return 0;
3182 }
3183
3184 VLIB_CLI_COMMAND (lookup_test_command, static) = {
3185     .path = "test lookup",
3186     .short_help = "test lookup",
3187     .function = test_lookup_command_fn,
3188 };
3189
3190 int vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
3191 {
3192   ip4_main_t * im4 = &ip4_main;
3193   ip4_fib_t * fib;
3194   uword * p = hash_get (im4->fib_index_by_table_id, table_id);
3195
3196   if (p == 0)
3197     return VNET_API_ERROR_NO_SUCH_FIB;
3198
3199   fib = vec_elt_at_index (im4->fibs, p[0]);
3200
3201   fib->flow_hash_config = flow_hash_config;
3202   return 0;
3203 }
3204
3205 static clib_error_t *
3206 set_ip_flow_hash_command_fn (vlib_main_t * vm,
3207                              unformat_input_t * input,
3208                              vlib_cli_command_t * cmd)
3209 {
3210   int matched = 0;
3211   u32 table_id = 0;
3212   u32 flow_hash_config = 0;
3213   int rv;
3214
3215   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3216     if (unformat (input, "table %d", &table_id))
3217       matched = 1;
3218 #define _(a,v) \
3219     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
3220     foreach_flow_hash_bit
3221 #undef _
3222     else break;
3223   }
3224
3225   if (matched == 0)
3226     return clib_error_return (0, "unknown input `%U'",
3227                               format_unformat_error, input);
3228
3229   rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
3230   switch (rv)
3231     {
3232     case 0:
3233       break;
3234
3235     case VNET_API_ERROR_NO_SUCH_FIB:
3236       return clib_error_return (0, "no such FIB table %d", table_id);
3237
3238     default:
3239       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3240       break;
3241     }
3242
3243   return 0;
3244 }
3245
3246 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) = {
3247   .path = "set ip flow-hash",
3248   .short_help =
3249   "set ip table flow-hash table <fib-id> src dst sport dport proto reverse",
3250   .function = set_ip_flow_hash_command_fn,
3251 };
3252
3253 int vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3254                                  u32 table_index)
3255 {
3256   vnet_main_t * vnm = vnet_get_main();
3257   vnet_interface_main_t * im = &vnm->interface_main;
3258   ip4_main_t * ipm = &ip4_main;
3259   ip_lookup_main_t * lm = &ipm->lookup_main;
3260   vnet_classify_main_t * cm = &vnet_classify_main;
3261
3262   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3263     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3264
3265   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3266     return VNET_API_ERROR_NO_SUCH_ENTRY;
3267
3268   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3269   lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
3270
3271   return 0;
3272 }
3273
3274 static clib_error_t *
3275 set_ip_classify_command_fn (vlib_main_t * vm,
3276                             unformat_input_t * input,
3277                             vlib_cli_command_t * cmd)
3278 {
3279   u32 table_index = ~0;
3280   int table_index_set = 0;
3281   u32 sw_if_index = ~0;
3282   int rv;
3283
3284   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3285     if (unformat (input, "table-index %d", &table_index))
3286       table_index_set = 1;
3287     else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3288                        vnet_get_main(), &sw_if_index))
3289       ;
3290     else
3291       break;
3292   }
3293
3294   if (table_index_set == 0)
3295     return clib_error_return (0, "classify table-index must be specified");
3296
3297   if (sw_if_index == ~0)
3298     return clib_error_return (0, "interface / subif must be specified");
3299
3300   rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
3301
3302   switch (rv)
3303     {
3304     case 0:
3305       break;
3306
3307     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3308       return clib_error_return (0, "No such interface");
3309
3310     case VNET_API_ERROR_NO_SUCH_ENTRY:
3311       return clib_error_return (0, "No such classifier table");
3312     }
3313   return 0;
3314 }
3315
3316 VLIB_CLI_COMMAND (set_ip_classify_command, static) = {
3317     .path = "set ip classify",
3318     .short_help =
3319     "set ip classify intfc <int> table-index <index>",
3320     .function = set_ip_classify_command_fn,
3321 };
3322