vnet/vnet/ip/ip4_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip4_forward.c: IP v4 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  43 #include <vnet/ethernet/arp_packet.h>   /* for ethernet_arp_header_t */
  44 #include <vnet/ppp/ppp.h>
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vnet/api_errno.h>     /* for API error numbers */
  47
  48 /* This is really, really simple but stupid fib. */
  49 u32
  50 ip4_fib_lookup_with_table (ip4_main_t * im, u32 fib_index,
  51                            ip4_address_t * dst,
  52                            u32 disable_default_route)
  53 {
  54   ip_lookup_main_t * lm = &im->lookup_main;
  55   ip4_fib_t * fib = vec_elt_at_index (im->fibs, fib_index);
  56   uword * p, * hash, key;
  57   i32 i, i_min, dst_address, ai;
  58
  59   i_min = disable_default_route ? 1 : 0;
  60   dst_address = clib_mem_unaligned (&dst->data_u32, u32);
  61   for (i = ARRAY_LEN (fib->adj_index_by_dst_address) - 1; i >= i_min; i--)
  62     {
  63       hash = fib->adj_index_by_dst_address[i];
  64       if (! hash)
  65         continue;
  66
  67       key = dst_address & im->fib_masks[i];
  68       if ((p = hash_get (hash, key)) != 0)
  69         {
  70           ai = p[0];
  71           goto done;
  72         }
  73     }
  74
  75   /* Nothing matches in table. */
  76   ai = lm->miss_adj_index;
  77
  78  done:
  79   return ai;
  80 }
  81
  82 static ip4_fib_t *
  83 create_fib_with_table_id (ip4_main_t * im, u32 table_id)
  84 {
  85   ip4_fib_t * fib;
  86   hash_set (im->fib_index_by_table_id, table_id, vec_len (im->fibs));
  87   vec_add2 (im->fibs, fib, 1);
  88   fib->table_id = table_id;
  89   fib->index = fib - im->fibs;
  90   fib->flow_hash_config = IP_FLOW_HASH_DEFAULT;
  91   fib->fwd_classify_table_index = ~0;
  92   fib->rev_classify_table_index = ~0;
  93   ip4_mtrie_init (&fib->mtrie);
  94   return fib;
  95 }
  96
  97 ip4_fib_t *
  98 find_ip4_fib_by_table_index_or_id (ip4_main_t * im,
  99                                    u32 table_index_or_id, u32 flags)
 100 {
 101   uword * p, fib_index;
 102
 103   fib_index = table_index_or_id;
 104   if (! (flags & IP4_ROUTE_FLAG_FIB_INDEX))
 105     {
 106       if (table_index_or_id == ~0) {
 107         table_index_or_id = 0;
 108         while ((p = hash_get (im->fib_index_by_table_id, table_index_or_id))) {
 109           table_index_or_id++;
 110         }
 111         return create_fib_with_table_id (im, table_index_or_id);
 112       }
 113
 114       p = hash_get (im->fib_index_by_table_id, table_index_or_id);
 115       if (! p)
 116         return create_fib_with_table_id (im, table_index_or_id);
 117       fib_index = p[0];
 118     }
 119   return vec_elt_at_index (im->fibs, fib_index);
 120 }
 121
 122 static void
 123 ip4_fib_init_adj_index_by_dst_address (ip_lookup_main_t * lm,
 124                                        ip4_fib_t * fib,
 125                                        u32 address_length)
 126 {
 127   hash_t * h;
 128   uword max_index;
 129
 130   ASSERT (lm->fib_result_n_bytes >= sizeof (uword));
 131   lm->fib_result_n_words = round_pow2 (lm->fib_result_n_bytes, sizeof (uword)) / sizeof (uword);
 132
 133   fib->adj_index_by_dst_address[address_length] =
 134     hash_create (32 /* elts */, lm->fib_result_n_words * sizeof (uword));
 135
 136   hash_set_flags (fib->adj_index_by_dst_address[address_length],
 137                   HASH_FLAG_NO_AUTO_SHRINK);
 138
 139   h = hash_header (fib->adj_index_by_dst_address[address_length]);
 140   max_index = (hash_value_bytes (h) / sizeof (fib->new_hash_values[0])) - 1;
 141
 142   /* Initialize new/old hash value vectors. */
 143   vec_validate_init_empty (fib->new_hash_values, max_index, ~0);
 144   vec_validate_init_empty (fib->old_hash_values, max_index, ~0);
 145 }
 146
 147 static void
 148 ip4_fib_set_adj_index (ip4_main_t * im,
 149                        ip4_fib_t * fib,
 150                        u32 flags,
 151                        u32 dst_address_u32,
 152                        u32 dst_address_length,
 153                        u32 adj_index)
 154 {
 155   ip_lookup_main_t * lm = &im->lookup_main;
 156   uword * hash;
 157
 158   if (vec_bytes(fib->old_hash_values))
 159     memset (fib->old_hash_values, ~0, vec_bytes (fib->old_hash_values));
 160   if (vec_bytes(fib->new_hash_values))
 161     memset (fib->new_hash_values, ~0, vec_bytes (fib->new_hash_values));
 162   fib->new_hash_values[0] = adj_index;
 163
 164   /* Make sure adj index is valid. */
 165   if (CLIB_DEBUG > 0)
 166     (void) ip_get_adjacency (lm, adj_index);
 167
 168   hash = fib->adj_index_by_dst_address[dst_address_length];
 169
 170   hash = _hash_set3 (hash, dst_address_u32,
 171                      fib->new_hash_values,
 172                      fib->old_hash_values);
 173
 174   fib->adj_index_by_dst_address[dst_address_length] = hash;
 175
 176   if (vec_len (im->add_del_route_callbacks) > 0)
 177     {
 178       ip4_add_del_route_callback_t * cb;
 179       ip4_address_t d;
 180       uword * p;
 181
 182       d.data_u32 = dst_address_u32;
 183       vec_foreach (cb, im->add_del_route_callbacks)
 184         if ((flags & cb->required_flags) == cb->required_flags)
 185           cb->function (im, cb->function_opaque,
 186                         fib, flags,
 187                         &d, dst_address_length,
 188                         fib->old_hash_values,
 189                         fib->new_hash_values);
 190
 191       p = hash_get (hash, dst_address_u32);
 192       clib_memcpy (p, fib->new_hash_values, vec_bytes (fib->new_hash_values));
 193     }
 194 }
 195
 196 void ip4_add_del_route (ip4_main_t * im, ip4_add_del_route_args_t * a)
 197 {
 198   ip_lookup_main_t * lm = &im->lookup_main;
 199   ip4_fib_t * fib;
 200   u32 dst_address, dst_address_length, adj_index, old_adj_index;
 201   uword * hash, is_del;
 202   ip4_add_del_route_callback_t * cb;
 203
 204   /* Either create new adjacency or use given one depending on arguments. */
 205   if (a->n_add_adj > 0)
 206     {
 207       ip_add_adjacency (lm, a->add_adj, a->n_add_adj, &adj_index);
 208       ip_call_add_del_adjacency_callbacks (lm, adj_index, /* is_del */ 0);
 209     }
 210   else
 211     adj_index = a->adj_index;
 212
 213   dst_address = a->dst_address.data_u32;
 214   dst_address_length = a->dst_address_length;
 215   fib = find_ip4_fib_by_table_index_or_id (im, a->table_index_or_table_id, a->flags);
 216
 217   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 218   dst_address &= im->fib_masks[dst_address_length];
 219
 220   if (! fib->adj_index_by_dst_address[dst_address_length])
 221     ip4_fib_init_adj_index_by_dst_address (lm, fib, dst_address_length);
 222
 223   hash = fib->adj_index_by_dst_address[dst_address_length];
 224
 225   is_del = (a->flags & IP4_ROUTE_FLAG_DEL) != 0;
 226
 227   if (is_del)
 228     {
 229       fib->old_hash_values[0] = ~0;
 230       hash = _hash_unset (hash, dst_address, fib->old_hash_values);
 231       fib->adj_index_by_dst_address[dst_address_length] = hash;
 232
 233       if (vec_len (im->add_del_route_callbacks) > 0
 234           && fib->old_hash_values[0] != ~0) /* make sure destination was found in hash */
 235         {
 236           fib->new_hash_values[0] = ~0;
 237           vec_foreach (cb, im->add_del_route_callbacks)
 238             if ((a->flags & cb->required_flags) == cb->required_flags)
 239               cb->function (im, cb->function_opaque,
 240                             fib, a->flags,
 241                             &a->dst_address, dst_address_length,
 242                             fib->old_hash_values,
 243                             fib->new_hash_values);
 244         }
 245     }
 246   else
 247     ip4_fib_set_adj_index (im, fib, a->flags, dst_address, dst_address_length,
 248                            adj_index);
 249
 250   old_adj_index = fib->old_hash_values[0];
 251
 252   /* Avoid spurious reference count increments */
 253   if (old_adj_index == adj_index
 254       && adj_index != ~0
 255       && !(a->flags & IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY))
 256     {
 257       ip_adjacency_t * adj = ip_get_adjacency (lm, adj_index);
 258       if (adj->share_count > 0)
 259         adj->share_count --;
 260     }
 261
 262   ip4_fib_mtrie_add_del_route (fib, a->dst_address, dst_address_length,
 263                                is_del ? old_adj_index : adj_index,
 264                                is_del);
 265
 266   /* Delete old adjacency index if present and changed. */
 267   if (! (a->flags & IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY)
 268       && old_adj_index != ~0
 269       && old_adj_index != adj_index)
 270     ip_del_adjacency (lm, old_adj_index);
 271 }
 272
 273 void
 274 ip4_add_del_route_next_hop (ip4_main_t * im,
 275                             u32 flags,
 276                             ip4_address_t * dst_address,
 277                             u32 dst_address_length,
 278                             ip4_address_t * next_hop,
 279                             u32 next_hop_sw_if_index,
 280                             u32 next_hop_weight, u32 adj_index,
 281                             u32 explicit_fib_index)
 282 {
 283   vnet_main_t * vnm = vnet_get_main();
 284   ip_lookup_main_t * lm = &im->lookup_main;
 285   u32 fib_index;
 286   ip4_fib_t * fib;
 287   u32 dst_address_u32, old_mp_adj_index, new_mp_adj_index;
 288   u32 dst_adj_index, nh_adj_index;
 289   uword * dst_hash, * dst_result;
 290   uword * nh_hash, * nh_result;
 291   ip_adjacency_t * dst_adj;
 292   ip_multipath_adjacency_t * old_mp, * new_mp;
 293   int is_del = (flags & IP4_ROUTE_FLAG_DEL) != 0;
 294   int is_interface_next_hop;
 295   clib_error_t * error = 0;
 296
 297   if (explicit_fib_index == (u32)~0)
 298       fib_index = vec_elt (im->fib_index_by_sw_if_index, next_hop_sw_if_index);
 299   else
 300       fib_index = explicit_fib_index;
 301
 302   fib = vec_elt_at_index (im->fibs, fib_index);
 303
 304   /* Lookup next hop to be added or deleted. */
 305   is_interface_next_hop = next_hop->data_u32 == 0;
 306   if (adj_index == (u32)~0)
 307     {
 308       if (is_interface_next_hop)
 309         {
 310           nh_result = hash_get (im->interface_route_adj_index_by_sw_if_index, next_hop_sw_if_index);
 311           if (nh_result)
 312             nh_adj_index = *nh_result;
 313           else
 314             {
 315               ip_adjacency_t * adj;
 316               adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 317                                       &nh_adj_index);
 318               ip4_adjacency_set_interface_route (vnm, adj, next_hop_sw_if_index, /* if_address_index */ ~0);
 319               ip_call_add_del_adjacency_callbacks (lm, nh_adj_index, /* is_del */ 0);
 320               hash_set (im->interface_route_adj_index_by_sw_if_index, next_hop_sw_if_index, nh_adj_index);
 321             }
 322         }
 323       else
 324         {
 325           nh_hash = fib->adj_index_by_dst_address[32];
 326           nh_result = hash_get (nh_hash, next_hop->data_u32);
 327
 328           /* Next hop must be known. */
 329           if (! nh_result)
 330             {
 331               ip_adjacency_t * adj;
 332
 333               nh_adj_index = ip4_fib_lookup_with_table (im, fib_index,
 334                                                         next_hop, 0);
 335               adj = ip_get_adjacency (lm, nh_adj_index);
 336               /* if ARP interface adjacencty is present, we need to
 337                  install ARP adjaceny for specific next hop */
 338               if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP &&
 339                   adj->arp.next_hop.ip4.as_u32 == 0)
 340                 {
 341                   nh_adj_index = vnet_arp_glean_add(fib_index, next_hop);
 342                 }
 343               else
 344                 {
 345                   /* Next hop is not known, so create indirect adj */
 346                   ip_adjacency_t add_adj;
 347                   add_adj.lookup_next_index = IP_LOOKUP_NEXT_INDIRECT;
 348                   add_adj.indirect.next_hop.ip4.as_u32 = next_hop->as_u32;
 349                   add_adj.explicit_fib_index = explicit_fib_index;
 350                   ip_add_adjacency (lm, &add_adj, 1, &nh_adj_index);
 351                 }
 352             }
 353           else
 354             nh_adj_index = *nh_result;
 355         }
 356     }
 357   else
 358     {
 359       nh_adj_index = adj_index;
 360     }
 361   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 362   dst_address_u32 = dst_address->data_u32 & im->fib_masks[dst_address_length];
 363
 364   dst_hash = fib->adj_index_by_dst_address[dst_address_length];
 365   dst_result = hash_get (dst_hash, dst_address_u32);
 366   if (dst_result)
 367     {
 368       dst_adj_index = dst_result[0];
 369       dst_adj = ip_get_adjacency (lm, dst_adj_index);
 370     }
 371   else
 372     {
 373       /* For deletes destination must be known. */
 374       if (is_del)
 375         {
 376           vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
 377           error = clib_error_return (0, "unknown destination %U/%d",
 378                                      format_ip4_address, dst_address,
 379                                      dst_address_length);
 380           goto done;
 381         }
 382
 383       dst_adj_index = ~0;
 384       dst_adj = 0;
 385     }
 386
 387   /* Ignore adds of X/32 with next hop of X. */
 388   if (! is_del
 389       && dst_address_length == 32
 390       && dst_address->data_u32 == next_hop->data_u32
 391       && adj_index != (u32)~0)
 392     {
 393       vnm->api_errno = VNET_API_ERROR_PREFIX_MATCHES_NEXT_HOP;
 394       error = clib_error_return (0, "prefix matches next hop %U/%d",
 395                                  format_ip4_address, dst_address,
 396                                  dst_address_length);
 397       goto done;
 398     }
 399
 400   /* Destination is not known and default weight is set so add route
 401      to existing non-multipath adjacency */
 402   if (dst_adj_index == ~0 && next_hop_weight == 1 && next_hop_sw_if_index == ~0)
 403     {
 404       /* create new adjacency */
 405       ip4_add_del_route_args_t a;
 406       a.table_index_or_table_id = fib_index;
 407       a.flags = ((is_del ? IP4_ROUTE_FLAG_DEL : IP4_ROUTE_FLAG_ADD)
 408                  | IP4_ROUTE_FLAG_FIB_INDEX
 409                  | IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 410                  | (flags & (IP4_ROUTE_FLAG_NO_REDISTRIBUTE
 411                              | IP4_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
 412       a.dst_address = dst_address[0];
 413       a.dst_address_length = dst_address_length;
 414       a.adj_index = nh_adj_index;
 415       a.add_adj = 0;
 416       a.n_add_adj = 0;
 417
 418       ip4_add_del_route (im, &a);
 419
 420       goto done;
 421     }
 422
 423   old_mp_adj_index = dst_adj ? dst_adj->heap_handle : ~0;
 424
 425   if (! ip_multipath_adjacency_add_del_next_hop
 426       (lm, is_del,
 427        old_mp_adj_index,
 428        nh_adj_index,
 429        next_hop_weight,
 430        &new_mp_adj_index))
 431     {
 432       vnm->api_errno = VNET_API_ERROR_NEXT_HOP_NOT_FOUND_MP;
 433       error = clib_error_return (0, "requested deleting next-hop %U not found in multi-path",
 434                                  format_ip4_address, next_hop);
 435       goto done;
 436     }
 437
 438   old_mp = new_mp = 0;
 439   if (old_mp_adj_index != ~0)
 440     old_mp = vec_elt_at_index (lm->multipath_adjacencies, old_mp_adj_index);
 441   if (new_mp_adj_index != ~0)
 442     new_mp = vec_elt_at_index (lm->multipath_adjacencies, new_mp_adj_index);
 443
 444   if (old_mp != new_mp)
 445     {
 446       ip4_add_del_route_args_t a;
 447       a.table_index_or_table_id = fib_index;
 448       a.flags = ((is_del && ! new_mp ? IP4_ROUTE_FLAG_DEL : IP4_ROUTE_FLAG_ADD)
 449                  | IP4_ROUTE_FLAG_FIB_INDEX
 450                  | IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 451                  | (flags & (IP4_ROUTE_FLAG_NO_REDISTRIBUTE | IP4_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
 452       a.dst_address = dst_address[0];
 453       a.dst_address_length = dst_address_length;
 454       a.adj_index = new_mp ? new_mp->adj_index : dst_adj_index;
 455       a.add_adj = 0;
 456       a.n_add_adj = 0;
 457
 458       ip4_add_del_route (im, &a);
 459     }
 460
 461  done:
 462   if (error)
 463     clib_error_report (error);
 464 }
 465
 466 void *
 467 ip4_get_route (ip4_main_t * im,
 468                u32 table_index_or_table_id,
 469                u32 flags,
 470                u8 * address,
 471                u32 address_length)
 472 {
 473   ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 474   u32 dst_address = * (u32 *) address;
 475   uword * hash, * p;
 476
 477   ASSERT (address_length < ARRAY_LEN (im->fib_masks));
 478   dst_address &= im->fib_masks[address_length];
 479
 480   hash = fib->adj_index_by_dst_address[address_length];
 481   p = hash_get (hash, dst_address);
 482   return (void *) p;
 483 }
 484
 485 void
 486 ip4_foreach_matching_route (ip4_main_t * im,
 487                             u32 table_index_or_table_id,
 488                             u32 flags,
 489                             ip4_address_t * address,
 490                             u32 address_length,
 491                             ip4_address_t ** results,
 492                             u8 ** result_lengths)
 493 {
 494   ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 495   u32 dst_address = address->data_u32;
 496   u32 this_length = address_length;
 497
 498   if (*results)
 499     _vec_len (*results) = 0;
 500   if (*result_lengths)
 501     _vec_len (*result_lengths) = 0;
 502
 503   while (this_length <= 32 && vec_len (results) == 0)
 504     {
 505       uword k, v;
 506       hash_foreach (k, v, fib->adj_index_by_dst_address[this_length], ({
 507         if (0 == ((k ^ dst_address) & im->fib_masks[address_length]))
 508           {
 509             ip4_address_t a;
 510             a.data_u32 = k;
 511             vec_add1 (*results, a);
 512             vec_add1 (*result_lengths, this_length);
 513           }
 514       }));
 515
 516       this_length++;
 517     }
 518 }
 519
 520 void ip4_maybe_remap_adjacencies (ip4_main_t * im,
 521                                   u32 table_index_or_table_id,
 522                                   u32 flags)
 523 {
 524   ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 525   ip_lookup_main_t * lm = &im->lookup_main;
 526   u32 i, l;
 527   ip4_address_t a;
 528   ip4_add_del_route_callback_t * cb;
 529   static ip4_address_t * to_delete;
 530
 531   if (lm->n_adjacency_remaps == 0)
 532     return;
 533
 534   for (l = 0; l <= 32; l++)
 535     {
 536       hash_pair_t * p;
 537       uword * hash = fib->adj_index_by_dst_address[l];
 538
 539       if (hash_elts (hash) == 0)
 540         continue;
 541
 542       if (to_delete)
 543         _vec_len (to_delete) = 0;
 544
 545       hash_foreach_pair (p, hash, ({
 546         u32 adj_index = p->value[0];
 547         u32 m = vec_elt (lm->adjacency_remap_table, adj_index);
 548
 549         if (m)
 550           {
 551             /* Record destination address from hash key. */
 552             a.data_u32 = p->key;
 553
 554             /* New adjacency points to nothing: so delete prefix. */
 555             if (m == ~0)
 556               vec_add1 (to_delete, a);
 557             else
 558               {
 559                 /* Remap to new adjacency. */
 560                 clib_memcpy (fib->old_hash_values, p->value, vec_bytes (fib->old_hash_values));
 561
 562                 /* Set new adjacency value. */
 563                 fib->new_hash_values[0] = p->value[0] = m - 1;
 564
 565                 vec_foreach (cb, im->add_del_route_callbacks)
 566                   if ((flags & cb->required_flags) == cb->required_flags)
 567                     cb->function (im, cb->function_opaque,
 568                                   fib, flags | IP4_ROUTE_FLAG_ADD,
 569                                   &a, l,
 570                                   fib->old_hash_values,
 571                                   fib->new_hash_values);
 572               }
 573           }
 574       }));
 575
 576       fib->new_hash_values[0] = ~0;
 577       for (i = 0; i < vec_len (to_delete); i++)
 578         {
 579           hash = _hash_unset (hash, to_delete[i].data_u32, fib->old_hash_values);
 580           vec_foreach (cb, im->add_del_route_callbacks)
 581             if ((flags & cb->required_flags) == cb->required_flags)
 582               cb->function (im, cb->function_opaque,
 583                             fib, flags | IP4_ROUTE_FLAG_DEL,
 584                             &a, l,
 585                             fib->old_hash_values,
 586                             fib->new_hash_values);
 587         }
 588     }
 589
 590   /* Also remap adjacencies in mtrie. */
 591   ip4_mtrie_maybe_remap_adjacencies (lm, &fib->mtrie);
 592
 593   /* Reset mapping table. */
 594   vec_zero (lm->adjacency_remap_table);
 595
 596   /* All remaps have been performed. */
 597   lm->n_adjacency_remaps = 0;
 598 }
 599
 600 void ip4_delete_matching_routes (ip4_main_t * im,
 601                                  u32 table_index_or_table_id,
 602                                  u32 flags,
 603                                  ip4_address_t * address,
 604                                  u32 address_length)
 605 {
 606   static ip4_address_t * matching_addresses;
 607   static u8 * matching_address_lengths;
 608   u32 l, i;
 609   ip4_add_del_route_args_t a;
 610
 611   a.flags = IP4_ROUTE_FLAG_DEL | IP4_ROUTE_FLAG_NO_REDISTRIBUTE | flags;
 612   a.table_index_or_table_id = table_index_or_table_id;
 613   a.adj_index = ~0;
 614   a.add_adj = 0;
 615   a.n_add_adj = 0;
 616
 617   for (l = address_length + 1; l <= 32; l++)
 618     {
 619       ip4_foreach_matching_route (im, table_index_or_table_id, flags,
 620                                   address,
 621                                   l,
 622                                   &matching_addresses,
 623                                   &matching_address_lengths);
 624       for (i = 0; i < vec_len (matching_addresses); i++)
 625         {
 626           a.dst_address = matching_addresses[i];
 627           a.dst_address_length = matching_address_lengths[i];
 628           ip4_add_del_route (im, &a);
 629         }
 630     }
 631
 632   ip4_maybe_remap_adjacencies (im, table_index_or_table_id, flags);
 633 }
 634
 635 void
 636 ip4_forward_next_trace (vlib_main_t * vm,
 637                         vlib_node_runtime_t * node,
 638                         vlib_frame_t * frame,
 639                         vlib_rx_or_tx_t which_adj_index);
 640
 641 always_inline uword
 642 ip4_lookup_inline (vlib_main_t * vm,
 643                    vlib_node_runtime_t * node,
 644                    vlib_frame_t * frame,
 645                    int lookup_for_responses_to_locally_received_packets,
 646                    int is_indirect)
 647 {
 648   ip4_main_t * im = &ip4_main;
 649   ip_lookup_main_t * lm = &im->lookup_main;
 650   vlib_combined_counter_main_t * cm = &im->lookup_main.adjacency_counters;
 651   u32 n_left_from, n_left_to_next, * from, * to_next;
 652   ip_lookup_next_t next;
 653   u32 cpu_index = os_get_cpu_number();
 654
 655   from = vlib_frame_vector_args (frame);
 656   n_left_from = frame->n_vectors;
 657   next = node->cached_next_index;
 658
 659   if (node->flags & VLIB_NODE_FLAG_TRACE)
 660       ip4_forward_next_trace(vm, node, frame, VLIB_TX);
 661
 662   while (n_left_from > 0)
 663     {
 664       vlib_get_next_frame (vm, node, next,
 665                            to_next, n_left_to_next);
 666
 667       while (n_left_from >= 4 && n_left_to_next >= 2)
 668         {
 669           vlib_buffer_t * p0, * p1;
 670           ip4_header_t * ip0, * ip1;
 671           __attribute__((unused)) tcp_header_t * tcp0, * tcp1;
 672           ip_lookup_next_t next0, next1;
 673           ip_adjacency_t * adj0, * adj1;
 674           ip4_fib_mtrie_t * mtrie0, * mtrie1;
 675           ip4_fib_mtrie_leaf_t leaf0, leaf1;
 676           ip4_address_t * dst_addr0, *dst_addr1;
 677           __attribute__((unused)) u32 pi0, fib_index0, adj_index0, is_tcp_udp0;
 678           __attribute__((unused)) u32 pi1, fib_index1, adj_index1, is_tcp_udp1;
 679           u32 flow_hash_config0, flow_hash_config1;
 680           u32 hash_c0, hash_c1;
 681           u32 wrong_next;
 682
 683           /* Prefetch next iteration. */
 684           {
 685             vlib_buffer_t * p2, * p3;
 686
 687             p2 = vlib_get_buffer (vm, from[2]);
 688             p3 = vlib_get_buffer (vm, from[3]);
 689
 690             vlib_prefetch_buffer_header (p2, LOAD);
 691             vlib_prefetch_buffer_header (p3, LOAD);
 692
 693             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
 694             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
 695           }
 696
 697           pi0 = to_next[0] = from[0];
 698           pi1 = to_next[1] = from[1];
 699
 700           p0 = vlib_get_buffer (vm, pi0);
 701           p1 = vlib_get_buffer (vm, pi1);
 702
 703           ip0 = vlib_buffer_get_current (p0);
 704           ip1 = vlib_buffer_get_current (p1);
 705
 706           if (is_indirect)
 707             {
 708               ip_adjacency_t * iadj0, * iadj1;
 709               iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
 710               iadj1 = ip_get_adjacency (lm, vnet_buffer(p1)->ip.adj_index[VLIB_TX]);
 711               dst_addr0 = &iadj0->indirect.next_hop.ip4;
 712               dst_addr1 = &iadj1->indirect.next_hop.ip4;
 713             }
 714           else
 715             {
 716               dst_addr0 = &ip0->dst_address;
 717               dst_addr1 = &ip1->dst_address;
 718             }
 719
 720           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 721           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 722           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 723             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 724           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
 725             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
 726
 727
 728           if (! lookup_for_responses_to_locally_received_packets)
 729             {
 730               mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
 731               mtrie1 = &vec_elt_at_index (im->fibs, fib_index1)->mtrie;
 732
 733               leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
 734
 735               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 0);
 736               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 0);
 737             }
 738
 739           tcp0 = (void *) (ip0 + 1);
 740           tcp1 = (void *) (ip1 + 1);
 741
 742           is_tcp_udp0 = (ip0->protocol == IP_PROTOCOL_TCP
 743                          || ip0->protocol == IP_PROTOCOL_UDP);
 744           is_tcp_udp1 = (ip1->protocol == IP_PROTOCOL_TCP
 745                          || ip1->protocol == IP_PROTOCOL_UDP);
 746
 747           if (! lookup_for_responses_to_locally_received_packets)
 748             {
 749               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
 750               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 1);
 751             }
 752
 753           if (! lookup_for_responses_to_locally_received_packets)
 754             {
 755               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 756               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
 757             }
 758
 759           if (! lookup_for_responses_to_locally_received_packets)
 760             {
 761               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 762               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
 763             }
 764
 765           if (lookup_for_responses_to_locally_received_packets)
 766             {
 767               adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 768               adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
 769             }
 770           else
 771             {
 772               /* Handle default route. */
 773               leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 774               leaf1 = (leaf1 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie1->default_leaf : leaf1);
 775
 776               adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 777               adj_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 778             }
 779
 780           ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
 781                                                            dst_addr0,
 782                                                            /* no_default_route */ 0));
 783           ASSERT (adj_index1 == ip4_fib_lookup_with_table (im, fib_index1,
 784                                                            dst_addr1,
 785                                                            /* no_default_route */ 0));
 786           adj0 = ip_get_adjacency (lm, adj_index0);
 787           adj1 = ip_get_adjacency (lm, adj_index1);
 788
 789           next0 = adj0->lookup_next_index;
 790           next1 = adj1->lookup_next_index;
 791
 792           /* Use flow hash to compute multipath adjacency. */
 793           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 794           hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
 795           if (PREDICT_FALSE (adj0->n_adj > 1))
 796             {
 797               flow_hash_config0 =
 798                 vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
 799               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 800                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 801             }
 802           if (PREDICT_FALSE(adj1->n_adj > 1))
 803             {
 804               flow_hash_config1 =
 805                 vec_elt_at_index (im->fibs, fib_index1)->flow_hash_config;
 806               hash_c1 = vnet_buffer (p1)->ip.flow_hash =
 807                 ip4_compute_flow_hash (ip1, flow_hash_config1);
 808             }
 809
 810           ASSERT (adj0->n_adj > 0);
 811           ASSERT (adj1->n_adj > 0);
 812           ASSERT (is_pow2 (adj0->n_adj));
 813           ASSERT (is_pow2 (adj1->n_adj));
 814           adj_index0 += (hash_c0 & (adj0->n_adj - 1));
 815           adj_index1 += (hash_c1 & (adj1->n_adj - 1));
 816
 817           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 818           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
 819
 820           vlib_increment_combined_counter
 821               (cm, cpu_index, adj_index0, 1,
 822                vlib_buffer_length_in_chain (vm, p0)
 823                + sizeof(ethernet_header_t));
 824           vlib_increment_combined_counter
 825               (cm, cpu_index, adj_index1, 1,
 826                vlib_buffer_length_in_chain (vm, p1)
 827                + sizeof(ethernet_header_t));
 828
 829           from += 2;
 830           to_next += 2;
 831           n_left_to_next -= 2;
 832           n_left_from -= 2;
 833
 834           wrong_next = (next0 != next) + 2*(next1 != next);
 835           if (PREDICT_FALSE (wrong_next != 0))
 836             {
 837               switch (wrong_next)
 838                 {
 839                 case 1:
 840                   /* A B A */
 841                   to_next[-2] = pi1;
 842                   to_next -= 1;
 843                   n_left_to_next += 1;
 844                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 845                   break;
 846
 847                 case 2:
 848                   /* A A B */
 849                   to_next -= 1;
 850                   n_left_to_next += 1;
 851                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 852                   break;
 853
 854                 case 3:
 855                   /* A B C */
 856                   to_next -= 2;
 857                   n_left_to_next += 2;
 858                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 859                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 860                   if (next0 == next1)
 861                     {
 862                       /* A B B */
 863                       vlib_put_next_frame (vm, node, next, n_left_to_next);
 864                       next = next1;
 865                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 866                     }
 867                 }
 868             }
 869         }
 870
 871       while (n_left_from > 0 && n_left_to_next > 0)
 872         {
 873           vlib_buffer_t * p0;
 874           ip4_header_t * ip0;
 875           __attribute__((unused)) tcp_header_t * tcp0;
 876           ip_lookup_next_t next0;
 877           ip_adjacency_t * adj0;
 878           ip4_fib_mtrie_t * mtrie0;
 879           ip4_fib_mtrie_leaf_t leaf0;
 880           ip4_address_t * dst_addr0;
 881           __attribute__((unused)) u32 pi0, fib_index0, adj_index0, is_tcp_udp0;
 882           u32 flow_hash_config0, hash_c0;
 883
 884           pi0 = from[0];
 885           to_next[0] = pi0;
 886
 887           p0 = vlib_get_buffer (vm, pi0);
 888
 889           ip0 = vlib_buffer_get_current (p0);
 890
 891           if (is_indirect)
 892             {
 893               ip_adjacency_t * iadj0;
 894               iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
 895               dst_addr0 = &iadj0->indirect.next_hop.ip4;
 896             }
 897           else
 898             {
 899               dst_addr0 = &ip0->dst_address;
 900             }
 901
 902           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 903           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 904             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 905
 906           if (! lookup_for_responses_to_locally_received_packets)
 907             {
 908               mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
 909
 910               leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
 911
 912               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 0);
 913             }
 914
 915           tcp0 = (void *) (ip0 + 1);
 916
 917           is_tcp_udp0 = (ip0->protocol == IP_PROTOCOL_TCP
 918                          || ip0->protocol == IP_PROTOCOL_UDP);
 919
 920           if (! lookup_for_responses_to_locally_received_packets)
 921             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
 922
 923           if (! lookup_for_responses_to_locally_received_packets)
 924             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 925
 926           if (! lookup_for_responses_to_locally_received_packets)
 927             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 928
 929           if (lookup_for_responses_to_locally_received_packets)
 930             adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 931           else
 932             {
 933               /* Handle default route. */
 934               leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 935               adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 936             }
 937
 938           ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
 939                                                            dst_addr0,
 940                                                            /* no_default_route */ 0));
 941
 942           adj0 = ip_get_adjacency (lm, adj_index0);
 943
 944           next0 = adj0->lookup_next_index;
 945
 946           /* Use flow hash to compute multipath adjacency. */
 947           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 948           if (PREDICT_FALSE(adj0->n_adj > 1))
 949             {
 950               flow_hash_config0 =
 951                 vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
 952
 953               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 954                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 955             }
 956
 957           ASSERT (adj0->n_adj > 0);
 958           ASSERT (is_pow2 (adj0->n_adj));
 959           adj_index0 += (hash_c0 & (adj0->n_adj - 1));
 960
 961           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 962
 963           vlib_increment_combined_counter
 964               (cm, cpu_index, adj_index0, 1,
 965                vlib_buffer_length_in_chain (vm, p0)
 966                + sizeof(ethernet_header_t));
 967
 968           from += 1;
 969           to_next += 1;
 970           n_left_to_next -= 1;
 971           n_left_from -= 1;
 972
 973           if (PREDICT_FALSE (next0 != next))
 974             {
 975               n_left_to_next += 1;
 976               vlib_put_next_frame (vm, node, next, n_left_to_next);
 977               next = next0;
 978               vlib_get_next_frame (vm, node, next,
 979                                    to_next, n_left_to_next);
 980               to_next[0] = pi0;
 981               to_next += 1;
 982               n_left_to_next -= 1;
 983             }
 984         }
 985
 986       vlib_put_next_frame (vm, node, next, n_left_to_next);
 987     }
 988
 989   return frame->n_vectors;
 990 }
 991
 992 static uword
 993 ip4_lookup (vlib_main_t * vm,
 994             vlib_node_runtime_t * node,
 995             vlib_frame_t * frame)
 996 {
 997   return ip4_lookup_inline (vm, node, frame,
 998                             /* lookup_for_responses_to_locally_received_packets */ 0,
 999                             /* is_indirect */ 0);
1000
1001 }
1002
1003 void ip4_adjacency_set_interface_route (vnet_main_t * vnm,
1004                                         ip_adjacency_t * adj,
1005                                         u32 sw_if_index,
1006                                         u32 if_address_index)
1007 {
1008   vnet_hw_interface_t * hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
1009   ip_lookup_next_t n;
1010   vnet_l3_packet_type_t packet_type;
1011   u32 node_index;
1012
1013   if (hw->hw_class_index == ethernet_hw_interface_class.index
1014       || hw->hw_class_index == srp_hw_interface_class.index)
1015     {
1016       /*
1017        * We have a bit of a problem in this case. ip4-arp uses
1018        * the rewrite_header.next_index to hand pkts to the
1019        * indicated inteface output node. We can end up in
1020        * ip4_rewrite_local, too, which also pays attention to
1021        * rewrite_header.next index. Net result: a hack in
1022        * ip4_rewrite_local...
1023        */
1024       n = IP_LOOKUP_NEXT_ARP;
1025       node_index = ip4_arp_node.index;
1026       adj->if_address_index = if_address_index;
1027       adj->arp.next_hop.ip4.as_u32 = 0;
1028       ip46_address_reset(&adj->arp.next_hop);
1029       packet_type = VNET_L3_PACKET_TYPE_ARP;
1030     }
1031   else
1032     {
1033       n = IP_LOOKUP_NEXT_REWRITE;
1034       node_index = ip4_rewrite_node.index;
1035       packet_type = VNET_L3_PACKET_TYPE_IP4;
1036     }
1037
1038   adj->lookup_next_index = n;
1039   vnet_rewrite_for_sw_interface
1040     (vnm,
1041      packet_type,
1042      sw_if_index,
1043      node_index,
1044      VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST,
1045      &adj->rewrite_header,
1046      sizeof (adj->rewrite_data));
1047 }
1048
1049 static void
1050 ip4_add_interface_routes (u32 sw_if_index,
1051                           ip4_main_t * im, u32 fib_index,
1052                           ip_interface_address_t * a)
1053 {
1054   vnet_main_t * vnm = vnet_get_main();
1055   ip_lookup_main_t * lm = &im->lookup_main;
1056   ip_adjacency_t * adj;
1057   ip4_address_t * address = ip_interface_address_get_address (lm, a);
1058   ip4_add_del_route_args_t x;
1059   vnet_hw_interface_t * hw_if = vnet_get_sup_hw_interface (vnm, sw_if_index);
1060   u32 classify_table_index;
1061
1062   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1063   x.table_index_or_table_id = fib_index;
1064   x.flags = (IP4_ROUTE_FLAG_ADD
1065              | IP4_ROUTE_FLAG_FIB_INDEX
1066              | IP4_ROUTE_FLAG_NO_REDISTRIBUTE);
1067   x.dst_address = address[0];
1068   x.dst_address_length = a->address_length;
1069   x.n_add_adj = 0;
1070   x.add_adj = 0;
1071
1072   a->neighbor_probe_adj_index = ~0;
1073   if (a->address_length < 32)
1074     {
1075       adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1076                               &x.adj_index);
1077       ip4_adjacency_set_interface_route (vnm, adj, sw_if_index, a - lm->if_address_pool);
1078       ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1079       ip4_add_del_route (im, &x);
1080       a->neighbor_probe_adj_index = x.adj_index;
1081     }
1082
1083   /* Add e.g. 1.1.1.1/32 as local to this host. */
1084   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1085                           &x.adj_index);
1086
1087   classify_table_index = ~0;
1088   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
1089     classify_table_index = lm->classify_table_index_by_sw_if_index [sw_if_index];
1090   if (classify_table_index != (u32) ~0)
1091     {
1092       adj->lookup_next_index = IP_LOOKUP_NEXT_CLASSIFY;
1093       adj->classify.table_index = classify_table_index;
1094     }
1095   else
1096     adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
1097
1098   adj->if_address_index = a - lm->if_address_pool;
1099   adj->rewrite_header.sw_if_index = sw_if_index;
1100   adj->rewrite_header.max_l3_packet_bytes = hw_if->max_l3_packet_bytes[VLIB_RX];
1101   /*
1102    * Local adjs are never to be rewritten. Spoofed pkts w/ src = dst = local
1103    * fail an RPF-ish check, but still go thru the rewrite code...
1104    */
1105   adj->rewrite_header.data_bytes = 0;
1106
1107   ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1108   x.dst_address_length = 32;
1109   ip4_add_del_route (im, &x);
1110 }
1111
1112 static void
1113 ip4_del_interface_routes (ip4_main_t * im, u32 fib_index, ip4_address_t * address, u32 address_length)
1114 {
1115   ip4_add_del_route_args_t x;
1116
1117   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1118   x.table_index_or_table_id = fib_index;
1119   x.flags = (IP4_ROUTE_FLAG_DEL
1120              | IP4_ROUTE_FLAG_FIB_INDEX
1121              | IP4_ROUTE_FLAG_NO_REDISTRIBUTE);
1122   x.dst_address = address[0];
1123   x.dst_address_length = address_length;
1124   x.adj_index = ~0;
1125   x.n_add_adj = 0;
1126   x.add_adj = 0;
1127
1128   if (address_length < 32)
1129     ip4_add_del_route (im, &x);
1130
1131   x.dst_address_length = 32;
1132   ip4_add_del_route (im, &x);
1133
1134   ip4_delete_matching_routes (im,
1135                               fib_index,
1136                               IP4_ROUTE_FLAG_FIB_INDEX,
1137                               address,
1138                               address_length);
1139 }
1140
1141 typedef struct {
1142     u32 sw_if_index;
1143     ip4_address_t address;
1144     u32 length;
1145 } ip4_interface_address_t;
1146
1147 static clib_error_t *
1148 ip4_add_del_interface_address_internal (vlib_main_t * vm,
1149                                         u32 sw_if_index,
1150                                         ip4_address_t * new_address,
1151                                         u32 new_length,
1152                                         u32 redistribute,
1153                                         u32 insert_routes,
1154                                         u32 is_del);
1155
1156 static clib_error_t *
1157 ip4_add_del_interface_address_internal (vlib_main_t * vm,
1158                                         u32 sw_if_index,
1159                                         ip4_address_t * address,
1160                                         u32 address_length,
1161                                         u32 redistribute,
1162                                         u32 insert_routes,
1163                                         u32 is_del)
1164 {
1165   vnet_main_t * vnm = vnet_get_main();
1166   ip4_main_t * im = &ip4_main;
1167   ip_lookup_main_t * lm = &im->lookup_main;
1168   clib_error_t * error = 0;
1169   u32 if_address_index, elts_before;
1170   ip4_address_fib_t ip4_af, * addr_fib = 0;
1171
1172   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1173   ip4_addr_fib_init (&ip4_af, address,
1174                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
1175   vec_add1 (addr_fib, ip4_af);
1176
1177   /* When adding an address check that it does not conflict with an existing address. */
1178   if (! is_del)
1179     {
1180       ip_interface_address_t * ia;
1181       foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
1182                                     0 /* honor unnumbered */,
1183       ({
1184         ip4_address_t * x = ip_interface_address_get_address (&im->lookup_main, ia);
1185
1186         if (ip4_destination_matches_route (im, address, x, ia->address_length)
1187             || ip4_destination_matches_route (im, x, address, address_length))
1188           return clib_error_create ("failed to add %U which conflicts with %U for interface %U",
1189                                     format_ip4_address_and_length, address, address_length,
1190                                     format_ip4_address_and_length, x, ia->address_length,
1191                                     format_vnet_sw_if_index_name, vnm, sw_if_index);
1192       }));
1193     }
1194
1195   elts_before = pool_elts (lm->if_address_pool);
1196
1197   error = ip_interface_address_add_del
1198     (lm,
1199      sw_if_index,
1200      addr_fib,
1201      address_length,
1202      is_del,
1203      &if_address_index);
1204   if (error)
1205     goto done;
1206
1207   if (vnet_sw_interface_is_admin_up (vnm, sw_if_index) && insert_routes)
1208     {
1209       if (is_del)
1210         ip4_del_interface_routes (im, ip4_af.fib_index, address,
1211                                   address_length);
1212
1213       else
1214           ip4_add_interface_routes (sw_if_index,
1215                                     im, ip4_af.fib_index,
1216                                     pool_elt_at_index
1217                                     (lm->if_address_pool, if_address_index));
1218     }
1219
1220   /* If pool did not grow/shrink: add duplicate address. */
1221   if (elts_before != pool_elts (lm->if_address_pool))
1222     {
1223       ip4_add_del_interface_address_callback_t * cb;
1224       vec_foreach (cb, im->add_del_interface_address_callbacks)
1225         cb->function (im, cb->function_opaque, sw_if_index,
1226                       address, address_length,
1227                       if_address_index,
1228                       is_del);
1229     }
1230
1231  done:
1232   vec_free (addr_fib);
1233   return error;
1234 }
1235
1236 clib_error_t *
1237 ip4_add_del_interface_address (vlib_main_t * vm, u32 sw_if_index,
1238                                ip4_address_t * address, u32 address_length,
1239                                u32 is_del)
1240 {
1241   return ip4_add_del_interface_address_internal
1242     (vm, sw_if_index, address, address_length,
1243      /* redistribute */ 1,
1244      /* insert_routes */ 1,
1245      is_del);
1246 }
1247
1248 static clib_error_t *
1249 ip4_sw_interface_admin_up_down (vnet_main_t * vnm,
1250                                 u32 sw_if_index,
1251                                 u32 flags)
1252 {
1253   ip4_main_t * im = &ip4_main;
1254   ip_interface_address_t * ia;
1255   ip4_address_t * a;
1256   u32 is_admin_up, fib_index;
1257
1258   /* Fill in lookup tables with default table (0). */
1259   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1260
1261   vec_validate_init_empty (im->lookup_main.if_address_pool_index_by_sw_if_index, sw_if_index, ~0);
1262
1263   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1264
1265   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
1266
1267   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
1268                                 0 /* honor unnumbered */,
1269   ({
1270     a = ip_interface_address_get_address (&im->lookup_main, ia);
1271     if (is_admin_up)
1272       ip4_add_interface_routes (sw_if_index,
1273                                 im, fib_index,
1274                                 ia);
1275     else
1276       ip4_del_interface_routes (im, fib_index,
1277                                 a, ia->address_length);
1278   }));
1279
1280   return 0;
1281 }
1282
1283 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip4_sw_interface_admin_up_down);
1284
1285 static clib_error_t *
1286 ip4_sw_interface_add_del (vnet_main_t * vnm,
1287                           u32 sw_if_index,
1288                           u32 is_add)
1289 {
1290   vlib_main_t * vm = vnm->vlib_main;
1291   ip4_main_t * im = &ip4_main;
1292   ip_lookup_main_t * lm = &im->lookup_main;
1293   u32 ci, cast;
1294
1295   for (cast = 0; cast < VNET_N_CAST; cast++)
1296     {
1297       ip_config_main_t * cm = &lm->rx_config_mains[cast];
1298       vnet_config_main_t * vcm = &cm->config_main;
1299
1300       if (! vcm->node_index_by_feature_index)
1301         {
1302           if (cast == VNET_UNICAST)
1303             {
1304               static char * start_nodes[] = { "ip4-input", "ip4-input-no-checksum", };
1305               static char * feature_nodes[] = {
1306                 [IP4_RX_FEATURE_CHECK_ACCESS] = "ip4-inacl",
1307                 [IP4_RX_FEATURE_SOURCE_CHECK_REACHABLE_VIA_RX] = "ip4-source-check-via-rx",
1308                 [IP4_RX_FEATURE_SOURCE_CHECK_REACHABLE_VIA_ANY] = "ip4-source-check-via-any",
1309                 [IP4_RX_FEATURE_IPSEC] = "ipsec-input-ip4",
1310                 [IP4_RX_FEATURE_VPATH] = "vpath-input-ip4",
1311                 [IP4_RX_FEATURE_LOOKUP] = "ip4-lookup",
1312               };
1313
1314               vnet_config_init (vm, vcm,
1315                                 start_nodes, ARRAY_LEN (start_nodes),
1316                                 feature_nodes, ARRAY_LEN (feature_nodes));
1317             }
1318           else
1319             {
1320               static char * start_nodes[] = { "ip4-input", "ip4-input-no-checksum", };
1321               static char * feature_nodes[] = {
1322                 [IP4_RX_FEATURE_VPATH] = "vpath-input-ip4",
1323                 [IP4_RX_FEATURE_LOOKUP] = "ip4-lookup-multicast",
1324               };
1325
1326               vnet_config_init (vm, vcm,
1327                                 start_nodes, ARRAY_LEN (start_nodes),
1328                                 feature_nodes, ARRAY_LEN (feature_nodes));
1329             }
1330         }
1331
1332       vec_validate_init_empty (cm->config_index_by_sw_if_index, sw_if_index, ~0);
1333       ci = cm->config_index_by_sw_if_index[sw_if_index];
1334
1335       if (is_add)
1336         ci = vnet_config_add_feature (vm, vcm,
1337                                       ci,
1338                                       IP4_RX_FEATURE_LOOKUP,
1339                                       /* config data */ 0,
1340                                       /* # bytes of config data */ 0);
1341       else
1342         ci = vnet_config_del_feature (vm, vcm,
1343                                       ci,
1344                                       IP4_RX_FEATURE_LOOKUP,
1345                                       /* config data */ 0,
1346                                       /* # bytes of config data */ 0);
1347
1348       cm->config_index_by_sw_if_index[sw_if_index] = ci;
1349     }
1350
1351   return /* no error */ 0;
1352 }
1353
1354 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
1355
1356 static u8 * format_ip4_forward_next_trace (u8 * s, va_list * args);
1357
1358 VLIB_REGISTER_NODE (ip4_lookup_node) = {
1359   .function = ip4_lookup,
1360   .name = "ip4-lookup",
1361   .vector_size = sizeof (u32),
1362
1363   .format_trace = format_ip4_forward_next_trace,
1364
1365   .n_next_nodes = IP_LOOKUP_N_NEXT,
1366   .next_nodes = IP4_LOOKUP_NEXT_NODES,
1367 };
1368
1369 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup)
1370
1371 static uword
1372 ip4_indirect (vlib_main_t * vm,
1373                vlib_node_runtime_t * node,
1374                vlib_frame_t * frame)
1375 {
1376   return ip4_lookup_inline (vm, node, frame,
1377                             /* lookup_for_responses_to_locally_received_packets */ 0,
1378                             /* is_indirect */ 1);
1379 }
1380
1381 VLIB_REGISTER_NODE (ip4_indirect_node) = {
1382   .function = ip4_indirect,
1383   .name = "ip4-indirect",
1384   .vector_size = sizeof (u32),
1385
1386   .format_trace = format_ip4_forward_next_trace,
1387
1388   .n_next_nodes = IP_LOOKUP_N_NEXT,
1389   .next_nodes = IP4_LOOKUP_NEXT_NODES,
1390 };
1391
1392 VLIB_NODE_FUNCTION_MULTIARCH (ip4_indirect_node, ip4_indirect)
1393
1394
1395 /* Global IP4 main. */
1396 ip4_main_t ip4_main;
1397
1398 clib_error_t *
1399 ip4_lookup_init (vlib_main_t * vm)
1400 {
1401   ip4_main_t * im = &ip4_main;
1402   uword i;
1403
1404   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
1405     {
1406       u32 m;
1407
1408       if (i < 32)
1409         m = pow2_mask (i) << (32 - i);
1410       else
1411         m = ~0;
1412       im->fib_masks[i] = clib_host_to_net_u32 (m);
1413     }
1414
1415   /* Create FIB with index 0 and table id of 0. */
1416   find_ip4_fib_by_table_index_or_id (im, /* table id */ 0, IP4_ROUTE_FLAG_TABLE_ID);
1417
1418   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
1419
1420   {
1421     pg_node_t * pn;
1422     pn = pg_get_node (ip4_lookup_node.index);
1423     pn->unformat_edit = unformat_pg_ip4_header;
1424   }
1425
1426   {
1427     ethernet_arp_header_t h;
1428
1429     memset (&h, 0, sizeof (h));
1430
1431     /* Set target ethernet address to all zeros. */
1432     memset (h.ip4_over_ethernet[1].ethernet, 0, sizeof (h.ip4_over_ethernet[1].ethernet));
1433
1434 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
1435 #define _8(f,v) h.f = v;
1436     _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
1437     _16 (l3_type, ETHERNET_TYPE_IP4);
1438     _8 (n_l2_address_bytes, 6);
1439     _8 (n_l3_address_bytes, 4);
1440     _16 (opcode, ETHERNET_ARP_OPCODE_request);
1441 #undef _16
1442 #undef _8
1443
1444     vlib_packet_template_init (vm,
1445                                &im->ip4_arp_request_packet_template,
1446                                /* data */ &h,
1447                                sizeof (h),
1448                                /* alloc chunk size */ 8,
1449                                "ip4 arp");
1450   }
1451
1452   return 0;
1453 }
1454
1455 VLIB_INIT_FUNCTION (ip4_lookup_init);
1456
1457 typedef struct {
1458   /* Adjacency taken. */
1459   u32 adj_index;
1460   u32 flow_hash;
1461   u32 fib_index;
1462
1463   /* Packet data, possibly *after* rewrite. */
1464   u8 packet_data[64 - 1*sizeof(u32)];
1465 } ip4_forward_next_trace_t;
1466
1467 static u8 * format_ip4_forward_next_trace (u8 * s, va_list * args)
1468 {
1469   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1470   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1471   ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1472   vnet_main_t * vnm = vnet_get_main();
1473   ip4_main_t * im = &ip4_main;
1474   ip_adjacency_t * adj;
1475   uword indent = format_get_indent (s);
1476   char *fib_or_interface = "fib";
1477   if ((node == vlib_get_node(vm, ip4_rewrite_node.index)) ||
1478       (node == vlib_get_node(vm, ip4_rewrite_local_node.index))) {
1479     fib_or_interface = "tx_sw_if_index";
1480   }
1481
1482   adj = ip_get_adjacency (&im->lookup_main, t->adj_index);
1483   s = format (s, "%s %d adj-idx %d : %U flow hash: 0x%08x",
1484               fib_or_interface,
1485               t->fib_index, t->adj_index, format_ip_adjacency,
1486               vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1487   switch (adj->lookup_next_index)
1488     {
1489     case IP_LOOKUP_NEXT_REWRITE:
1490       s = format (s, "\n%U%U",
1491                   format_white_space, indent,
1492                   format_ip_adjacency_packet_data,
1493                   vnm, &im->lookup_main, t->adj_index,
1494                   t->packet_data, sizeof (t->packet_data));
1495       break;
1496
1497     default:
1498       break;
1499     }
1500
1501   return s;
1502 }
1503
1504 /* Common trace function for all ip4-forward next nodes. */
1505 void
1506 ip4_forward_next_trace (vlib_main_t * vm,
1507                         vlib_node_runtime_t * node,
1508                         vlib_frame_t * frame,
1509                         vlib_rx_or_tx_t which_adj_index)
1510 {
1511   u32 * from, n_left;
1512   ip4_main_t * im = &ip4_main;
1513
1514   n_left = frame->n_vectors;
1515   from = vlib_frame_vector_args (frame);
1516
1517   while (n_left >= 4)
1518     {
1519       u32 bi0, bi1;
1520       vlib_buffer_t * b0, * b1;
1521       ip4_forward_next_trace_t * t0, * t1;
1522
1523       /* Prefetch next iteration. */
1524       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1525       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1526
1527       bi0 = from[0];
1528       bi1 = from[1];
1529
1530       b0 = vlib_get_buffer (vm, bi0);
1531       b1 = vlib_get_buffer (vm, bi1);
1532
1533       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1534         {
1535           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1536           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1537           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1538           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1539               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1540               vec_elt (im->fib_index_by_sw_if_index,
1541                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1542
1543           clib_memcpy (t0->packet_data,
1544                   vlib_buffer_get_current (b0),
1545                   sizeof (t0->packet_data));
1546         }
1547       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1548         {
1549           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1550           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1551           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1552           t1->fib_index = (vnet_buffer(b1)->sw_if_index[VLIB_TX] != (u32)~0) ?
1553               vnet_buffer(b1)->sw_if_index[VLIB_TX] :
1554               vec_elt (im->fib_index_by_sw_if_index,
1555                        vnet_buffer(b1)->sw_if_index[VLIB_RX]);
1556           clib_memcpy (t1->packet_data,
1557                   vlib_buffer_get_current (b1),
1558                   sizeof (t1->packet_data));
1559         }
1560       from += 2;
1561       n_left -= 2;
1562     }
1563
1564   while (n_left >= 1)
1565     {
1566       u32 bi0;
1567       vlib_buffer_t * b0;
1568       ip4_forward_next_trace_t * t0;
1569
1570       bi0 = from[0];
1571
1572       b0 = vlib_get_buffer (vm, bi0);
1573
1574       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1575         {
1576           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1577           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1578           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1579           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1580               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1581               vec_elt (im->fib_index_by_sw_if_index,
1582                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1583           clib_memcpy (t0->packet_data,
1584                   vlib_buffer_get_current (b0),
1585                   sizeof (t0->packet_data));
1586         }
1587       from += 1;
1588       n_left -= 1;
1589     }
1590 }
1591
1592 static uword
1593 ip4_drop_or_punt (vlib_main_t * vm,
1594                   vlib_node_runtime_t * node,
1595                   vlib_frame_t * frame,
1596                   ip4_error_t error_code)
1597 {
1598   u32 * buffers = vlib_frame_vector_args (frame);
1599   uword n_packets = frame->n_vectors;
1600
1601   vlib_error_drop_buffers (vm, node,
1602                            buffers,
1603                            /* stride */ 1,
1604                            n_packets,
1605                            /* next */ 0,
1606                            ip4_input_node.index,
1607                            error_code);
1608
1609   if (node->flags & VLIB_NODE_FLAG_TRACE)
1610     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1611
1612   return n_packets;
1613 }
1614
1615 static uword
1616 ip4_drop (vlib_main_t * vm,
1617           vlib_node_runtime_t * node,
1618           vlib_frame_t * frame)
1619 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_DROP); }
1620
1621 static uword
1622 ip4_punt (vlib_main_t * vm,
1623           vlib_node_runtime_t * node,
1624           vlib_frame_t * frame)
1625 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_PUNT); }
1626
1627 static uword
1628 ip4_miss (vlib_main_t * vm,
1629           vlib_node_runtime_t * node,
1630           vlib_frame_t * frame)
1631 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_DST_LOOKUP_MISS); }
1632
1633 VLIB_REGISTER_NODE (ip4_drop_node,static) = {
1634   .function = ip4_drop,
1635   .name = "ip4-drop",
1636   .vector_size = sizeof (u32),
1637
1638   .format_trace = format_ip4_forward_next_trace,
1639
1640   .n_next_nodes = 1,
1641   .next_nodes = {
1642     [0] = "error-drop",
1643   },
1644 };
1645
1646 VLIB_NODE_FUNCTION_MULTIARCH (ip4_drop_node, ip4_drop)
1647
1648 VLIB_REGISTER_NODE (ip4_punt_node,static) = {
1649   .function = ip4_punt,
1650   .name = "ip4-punt",
1651   .vector_size = sizeof (u32),
1652
1653   .format_trace = format_ip4_forward_next_trace,
1654
1655   .n_next_nodes = 1,
1656   .next_nodes = {
1657     [0] = "error-punt",
1658   },
1659 };
1660
1661 VLIB_NODE_FUNCTION_MULTIARCH (ip4_punt_node, ip4_punt)
1662
1663 VLIB_REGISTER_NODE (ip4_miss_node,static) = {
1664   .function = ip4_miss,
1665   .name = "ip4-miss",
1666   .vector_size = sizeof (u32),
1667
1668   .format_trace = format_ip4_forward_next_trace,
1669
1670   .n_next_nodes = 1,
1671   .next_nodes = {
1672     [0] = "error-drop",
1673   },
1674 };
1675
1676 VLIB_NODE_FUNCTION_MULTIARCH (ip4_miss_node, ip4_miss)
1677
1678 /* Compute TCP/UDP/ICMP4 checksum in software. */
1679 u16
1680 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1681                               ip4_header_t * ip0)
1682 {
1683   ip_csum_t sum0;
1684   u32 ip_header_length, payload_length_host_byte_order;
1685   u32 n_this_buffer, n_bytes_left;
1686   u16 sum16;
1687   void * data_this_buffer;
1688
1689   /* Initialize checksum with ip header. */
1690   ip_header_length = ip4_header_bytes (ip0);
1691   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->length) - ip_header_length;
1692   sum0 = clib_host_to_net_u32 (payload_length_host_byte_order + (ip0->protocol << 16));
1693
1694   if (BITS (uword) == 32)
1695     {
1696       sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u32));
1697       sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->dst_address, u32));
1698     }
1699   else
1700     sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1701
1702   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1703   data_this_buffer = (void *) ip0 + ip_header_length;
1704   if (n_this_buffer + ip_header_length > p0->current_length)
1705     n_this_buffer = p0->current_length > ip_header_length ? p0->current_length - ip_header_length : 0;
1706   while (1)
1707     {
1708       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1709       n_bytes_left -= n_this_buffer;
1710       if (n_bytes_left == 0)
1711         break;
1712
1713       ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1714       p0 = vlib_get_buffer (vm, p0->next_buffer);
1715       data_this_buffer = vlib_buffer_get_current (p0);
1716       n_this_buffer = p0->current_length;
1717     }
1718
1719   sum16 = ~ ip_csum_fold (sum0);
1720
1721   return sum16;
1722 }
1723
1724 static u32
1725 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1726 {
1727   ip4_header_t * ip0 = vlib_buffer_get_current (p0);
1728   udp_header_t * udp0;
1729   u16 sum16;
1730
1731   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1732           || ip0->protocol == IP_PROTOCOL_UDP);
1733
1734   udp0 = (void *) (ip0 + 1);
1735   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1736     {
1737       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1738                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1739       return p0->flags;
1740     }
1741
1742   sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1743
1744   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1745                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1746
1747   return p0->flags;
1748 }
1749
1750 static uword
1751 ip4_local (vlib_main_t * vm,
1752            vlib_node_runtime_t * node,
1753            vlib_frame_t * frame)
1754 {
1755   ip4_main_t * im = &ip4_main;
1756   ip_lookup_main_t * lm = &im->lookup_main;
1757   ip_local_next_t next_index;
1758   u32 * from, * to_next, n_left_from, n_left_to_next;
1759   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip4_input_node.index);
1760
1761   from = vlib_frame_vector_args (frame);
1762   n_left_from = frame->n_vectors;
1763   next_index = node->cached_next_index;
1764
1765   if (node->flags & VLIB_NODE_FLAG_TRACE)
1766     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1767
1768   while (n_left_from > 0)
1769     {
1770       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1771
1772       while (n_left_from >= 4 && n_left_to_next >= 2)
1773         {
1774           vlib_buffer_t * p0, * p1;
1775           ip4_header_t * ip0, * ip1;
1776           udp_header_t * udp0, * udp1;
1777           ip4_fib_mtrie_t * mtrie0, * mtrie1;
1778           ip4_fib_mtrie_leaf_t leaf0, leaf1;
1779           ip_adjacency_t * adj0, * adj1;
1780           u32 pi0, ip_len0, udp_len0, flags0, next0, fib_index0, adj_index0;
1781           u32 pi1, ip_len1, udp_len1, flags1, next1, fib_index1, adj_index1;
1782           i32 len_diff0, len_diff1;
1783           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1784           u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1785           u8 enqueue_code;
1786
1787           pi0 = to_next[0] = from[0];
1788           pi1 = to_next[1] = from[1];
1789           from += 2;
1790           n_left_from -= 2;
1791           to_next += 2;
1792           n_left_to_next -= 2;
1793
1794           p0 = vlib_get_buffer (vm, pi0);
1795           p1 = vlib_get_buffer (vm, pi1);
1796
1797           ip0 = vlib_buffer_get_current (p0);
1798           ip1 = vlib_buffer_get_current (p1);
1799
1800           fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
1801                                 vnet_buffer(p0)->sw_if_index[VLIB_RX]);
1802           fib_index1 = vec_elt (im->fib_index_by_sw_if_index,
1803                                 vnet_buffer(p1)->sw_if_index[VLIB_RX]);
1804
1805           mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
1806           mtrie1 = &vec_elt_at_index (im->fibs, fib_index1)->mtrie;
1807
1808           leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
1809
1810           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 0);
1811           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 0);
1812
1813           /* Treat IP frag packets as "experimental" protocol for now
1814              until support of IP frag reassembly is implemented */
1815           proto0 = ip4_is_fragment(ip0) ? 0xfe : ip0->protocol;
1816           proto1 = ip4_is_fragment(ip1) ? 0xfe : ip1->protocol;
1817           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1818           is_udp1 = proto1 == IP_PROTOCOL_UDP;
1819           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1820           is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1821
1822           flags0 = p0->flags;
1823           flags1 = p1->flags;
1824
1825           good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1826           good_tcp_udp1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1827
1828           udp0 = ip4_next_header (ip0);
1829           udp1 = ip4_next_header (ip1);
1830
1831           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1832           good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1833           good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1834
1835           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
1836           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 1);
1837
1838           /* Verify UDP length. */
1839           ip_len0 = clib_net_to_host_u16 (ip0->length);
1840           ip_len1 = clib_net_to_host_u16 (ip1->length);
1841           udp_len0 = clib_net_to_host_u16 (udp0->length);
1842           udp_len1 = clib_net_to_host_u16 (udp1->length);
1843
1844           len_diff0 = ip_len0 - udp_len0;
1845           len_diff1 = ip_len1 - udp_len1;
1846
1847           len_diff0 = is_udp0 ? len_diff0 : 0;
1848           len_diff1 = is_udp1 ? len_diff1 : 0;
1849
1850           if (PREDICT_FALSE (! (is_tcp_udp0 & is_tcp_udp1
1851                                 & good_tcp_udp0 & good_tcp_udp1)))
1852             {
1853               if (is_tcp_udp0)
1854                 {
1855                   if (is_tcp_udp0
1856                       && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1857                     flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
1858                   good_tcp_udp0 =
1859                     (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1860                   good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1861                 }
1862               if (is_tcp_udp1)
1863                 {
1864                   if (is_tcp_udp1
1865                       && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1866                     flags1 = ip4_tcp_udp_validate_checksum (vm, p1);
1867                   good_tcp_udp1 =
1868                     (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1869                   good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1870                 }
1871             }
1872
1873           good_tcp_udp0 &= len_diff0 >= 0;
1874           good_tcp_udp1 &= len_diff1 >= 0;
1875
1876           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1877           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 2);
1878
1879           error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1880
1881           error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
1882           error1 = len_diff1 < 0 ? IP4_ERROR_UDP_LENGTH : error1;
1883
1884           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1885           error0 = (is_tcp_udp0 && ! good_tcp_udp0
1886                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0
1887                     : error0);
1888           error1 = (is_tcp_udp1 && ! good_tcp_udp1
1889                     ? IP4_ERROR_TCP_CHECKSUM + is_udp1
1890                     : error1);
1891
1892           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1893           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 3);
1894
1895           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1896           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
1897
1898           vnet_buffer (p1)->ip.adj_index[VLIB_RX] = adj_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
1899           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
1900
1901           ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
1902                                                            &ip0->src_address,
1903                                                            /* no_default_route */ 1));
1904           ASSERT (adj_index1 == ip4_fib_lookup_with_table (im, fib_index1,
1905                                                            &ip1->src_address,
1906                                                            /* no_default_route */ 1));
1907
1908           adj0 = ip_get_adjacency (lm, adj_index0);
1909           adj1 = ip_get_adjacency (lm, adj_index1);
1910
1911           /*
1912            * Must have a route to source otherwise we drop the packet.
1913            * ip4 broadcasts are accepted, e.g. to make dhcp client work
1914            */
1915           error0 = (error0 == IP4_ERROR_UNKNOWN_PROTOCOL
1916                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_REWRITE
1917                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP
1918                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_LOCAL
1919                     && ip0->dst_address.as_u32 != 0xFFFFFFFF
1920                     ? IP4_ERROR_SRC_LOOKUP_MISS
1921                     : error0);
1922           error1 = (error1 == IP4_ERROR_UNKNOWN_PROTOCOL
1923                     && adj1->lookup_next_index != IP_LOOKUP_NEXT_REWRITE
1924                     && adj1->lookup_next_index != IP_LOOKUP_NEXT_ARP
1925                     && adj1->lookup_next_index != IP_LOOKUP_NEXT_LOCAL
1926                     && ip0->dst_address.as_u32 != 0xFFFFFFFF
1927                     ? IP4_ERROR_SRC_LOOKUP_MISS
1928                     : error1);
1929
1930           next0 = lm->local_next_by_ip_protocol[proto0];
1931           next1 = lm->local_next_by_ip_protocol[proto1];
1932
1933           next0 = error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1934           next1 = error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1935
1936           p0->error = error0 ? error_node->errors[error0] : 0;
1937           p1->error = error1 ? error_node->errors[error1] : 0;
1938
1939           enqueue_code = (next0 != next_index) + 2*(next1 != next_index);
1940
1941           if (PREDICT_FALSE (enqueue_code != 0))
1942             {
1943               switch (enqueue_code)
1944                 {
1945                 case 1:
1946                   /* A B A */
1947                   to_next[-2] = pi1;
1948                   to_next -= 1;
1949                   n_left_to_next += 1;
1950                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
1951                   break;
1952
1953                 case 2:
1954                   /* A A B */
1955                   to_next -= 1;
1956                   n_left_to_next += 1;
1957                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
1958                   break;
1959
1960                 case 3:
1961                   /* A B B or A B C */
1962                   to_next -= 2;
1963                   n_left_to_next += 2;
1964                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
1965                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
1966                   if (next0 == next1)
1967                     {
1968                       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1969                       next_index = next1;
1970                       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1971                     }
1972                   break;
1973                 }
1974             }
1975         }
1976
1977       while (n_left_from > 0 && n_left_to_next > 0)
1978         {
1979           vlib_buffer_t * p0;
1980           ip4_header_t * ip0;
1981           udp_header_t * udp0;
1982           ip4_fib_mtrie_t * mtrie0;
1983           ip4_fib_mtrie_leaf_t leaf0;
1984           ip_adjacency_t * adj0;
1985           u32 pi0, next0, ip_len0, udp_len0, flags0, fib_index0, adj_index0;
1986           i32 len_diff0;
1987           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1988
1989           pi0 = to_next[0] = from[0];
1990           from += 1;
1991           n_left_from -= 1;
1992           to_next += 1;
1993           n_left_to_next -= 1;
1994
1995           p0 = vlib_get_buffer (vm, pi0);
1996
1997           ip0 = vlib_buffer_get_current (p0);
1998
1999           fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
2000                                 vnet_buffer(p0)->sw_if_index[VLIB_RX]);
2001
2002           mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
2003
2004           leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
2005
2006           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 0);
2007
2008           /* Treat IP frag packets as "experimental" protocol for now
2009              until support of IP frag reassembly is implemented */
2010           proto0 = ip4_is_fragment(ip0) ? 0xfe : ip0->protocol;
2011           is_udp0 = proto0 == IP_PROTOCOL_UDP;
2012           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
2013
2014           flags0 = p0->flags;
2015
2016           good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
2017
2018           udp0 = ip4_next_header (ip0);
2019
2020           /* Don't verify UDP checksum for packets with explicit zero checksum. */
2021           good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
2022
2023           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
2024
2025           /* Verify UDP length. */
2026           ip_len0 = clib_net_to_host_u16 (ip0->length);
2027           udp_len0 = clib_net_to_host_u16 (udp0->length);
2028
2029           len_diff0 = ip_len0 - udp_len0;
2030
2031           len_diff0 = is_udp0 ? len_diff0 : 0;
2032
2033           if (PREDICT_FALSE (! (is_tcp_udp0 & good_tcp_udp0)))
2034             {
2035               if (is_tcp_udp0)
2036                 {
2037                   if (is_tcp_udp0
2038                       && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
2039                     flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
2040                   good_tcp_udp0 =
2041                     (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
2042                   good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
2043                 }
2044             }
2045
2046           good_tcp_udp0 &= len_diff0 >= 0;
2047
2048           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
2049
2050           error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
2051
2052           error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
2053
2054           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
2055           error0 = (is_tcp_udp0 && ! good_tcp_udp0
2056                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0
2057                     : error0);
2058
2059           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
2060
2061           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2062           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
2063
2064           ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
2065                                                            &ip0->src_address,
2066                                                            /* no_default_route */ 1));
2067
2068           adj0 = ip_get_adjacency (lm, adj_index0);
2069
2070           /* Must have a route to source otherwise we drop the packet. */
2071           error0 = (error0 == IP4_ERROR_UNKNOWN_PROTOCOL
2072                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_REWRITE
2073                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP
2074                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_LOCAL
2075                     && ip0->dst_address.as_u32 != 0xFFFFFFFF
2076                     ? IP4_ERROR_SRC_LOOKUP_MISS
2077                     : error0);
2078
2079           next0 = lm->local_next_by_ip_protocol[proto0];
2080
2081           next0 = error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
2082
2083           p0->error = error0? error_node->errors[error0] : 0;
2084
2085           if (PREDICT_FALSE (next0 != next_index))
2086             {
2087               n_left_to_next += 1;
2088               vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2089
2090               next_index = next0;
2091               vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2092               to_next[0] = pi0;
2093               to_next += 1;
2094               n_left_to_next -= 1;
2095             }
2096         }
2097
2098       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2099     }
2100
2101   return frame->n_vectors;
2102 }
2103
2104 VLIB_REGISTER_NODE (ip4_local_node,static) = {
2105   .function = ip4_local,
2106   .name = "ip4-local",
2107   .vector_size = sizeof (u32),
2108
2109   .format_trace = format_ip4_forward_next_trace,
2110
2111   .n_next_nodes = IP_LOCAL_N_NEXT,
2112   .next_nodes = {
2113     [IP_LOCAL_NEXT_DROP] = "error-drop",
2114     [IP_LOCAL_NEXT_PUNT] = "error-punt",
2115     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
2116     [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",
2117   },
2118 };
2119
2120 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local)
2121
2122 void ip4_register_protocol (u32 protocol, u32 node_index)
2123 {
2124   vlib_main_t * vm = vlib_get_main();
2125   ip4_main_t * im = &ip4_main;
2126   ip_lookup_main_t * lm = &im->lookup_main;
2127
2128   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
2129   lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip4_local_node.index, node_index);
2130 }
2131
2132 static clib_error_t *
2133 show_ip_local_command_fn (vlib_main_t * vm,
2134                           unformat_input_t * input,
2135                          vlib_cli_command_t * cmd)
2136 {
2137   ip4_main_t * im = &ip4_main;
2138   ip_lookup_main_t * lm = &im->lookup_main;
2139   int i;
2140
2141   vlib_cli_output (vm, "Protocols handled by ip4_local");
2142   for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
2143     {
2144       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
2145         vlib_cli_output (vm, "%d", i);
2146     }
2147   return 0;
2148 }
2149
2150
2151
2152 VLIB_CLI_COMMAND (show_ip_local, static) = {
2153   .path = "show ip local",
2154   .function = show_ip_local_command_fn,
2155   .short_help = "Show ip local protocol table",
2156 };
2157
2158 static uword
2159 ip4_arp (vlib_main_t * vm,
2160          vlib_node_runtime_t * node,
2161          vlib_frame_t * frame)
2162 {
2163   vnet_main_t * vnm = vnet_get_main();
2164   ip4_main_t * im = &ip4_main;
2165   ip_lookup_main_t * lm = &im->lookup_main;
2166   u32 * from, * to_next_drop;
2167   uword n_left_from, n_left_to_next_drop, next_index;
2168   static f64 time_last_seed_change = -1e100;
2169   static u32 hash_seeds[3];
2170   static uword hash_bitmap[256 / BITS (uword)];
2171   f64 time_now;
2172
2173   if (node->flags & VLIB_NODE_FLAG_TRACE)
2174     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2175
2176   time_now = vlib_time_now (vm);
2177   if (time_now - time_last_seed_change > 1e-3)
2178     {
2179       uword i;
2180       u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
2181                                              sizeof (hash_seeds));
2182       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
2183         hash_seeds[i] = r[i];
2184
2185       /* Mark all hash keys as been no-seen before. */
2186       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
2187         hash_bitmap[i] = 0;
2188
2189       time_last_seed_change = time_now;
2190     }
2191
2192   from = vlib_frame_vector_args (frame);
2193   n_left_from = frame->n_vectors;
2194   next_index = node->cached_next_index;
2195   if (next_index == IP4_ARP_NEXT_DROP)
2196     next_index = IP4_ARP_N_NEXT; /* point to first interface */
2197
2198   while (n_left_from > 0)
2199     {
2200       vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
2201                            to_next_drop, n_left_to_next_drop);
2202
2203       while (n_left_from > 0 && n_left_to_next_drop > 0)
2204         {
2205           vlib_buffer_t * p0;
2206           ip4_header_t * ip0;
2207           ethernet_header_t * eh0;
2208           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
2209           uword bm0;
2210           ip_adjacency_t * adj0;
2211
2212           pi0 = from[0];
2213
2214           p0 = vlib_get_buffer (vm, pi0);
2215
2216           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2217           adj0 = ip_get_adjacency (lm, adj_index0);
2218           ip0 = vlib_buffer_get_current (p0);
2219
2220           /* If packet destination is not local, send ARP to next hop */
2221           if (adj0->arp.next_hop.ip4.as_u32)
2222             ip0->dst_address.data_u32 = adj0->arp.next_hop.ip4.as_u32;
2223
2224           /*
2225            * if ip4_rewrite_local applied the IP_LOOKUP_NEXT_ARP
2226            * rewrite to this packet, we need to skip it here.
2227            * Note, to distinguish from src IP addr *.8.6.*, we
2228            * check for a bcast eth dest instead of IPv4 version.
2229            */
2230           eh0 = (ethernet_header_t*)ip0;
2231           if ((ip0->ip_version_and_header_length & 0xF0) != 0x40)
2232             {
2233               u32 vlan_num = 0;
2234               u16 * etype = &eh0->type;
2235               while ((*etype == clib_host_to_net_u16 (0x8100)) //dot1q
2236                   || (*etype == clib_host_to_net_u16 (0x88a8)))//dot1ad
2237                 {
2238                   vlan_num += 1;
2239                   etype += 2; //vlan tag also 16 bits, same as etype
2240                 }
2241               if (*etype == clib_host_to_net_u16 (0x0806))     //arp
2242                 {
2243                   vlib_buffer_advance (
2244                       p0, sizeof(ethernet_header_t) + (4*vlan_num));
2245                   ip0 = vlib_buffer_get_current (p0);
2246                 }
2247             }
2248
2249           a0 = hash_seeds[0];
2250           b0 = hash_seeds[1];
2251           c0 = hash_seeds[2];
2252
2253           sw_if_index0 = adj0->rewrite_header.sw_if_index;
2254           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
2255
2256           a0 ^= ip0->dst_address.data_u32;
2257           b0 ^= sw_if_index0;
2258
2259           hash_v3_finalize32 (a0, b0, c0);
2260
2261           c0 &= BITS (hash_bitmap) - 1;
2262           c0 = c0 / BITS (uword);
2263           m0 = (uword) 1 << (c0 % BITS (uword));
2264
2265           bm0 = hash_bitmap[c0];
2266           drop0 = (bm0 & m0) != 0;
2267
2268           /* Mark it as seen. */
2269           hash_bitmap[c0] = bm0 | m0;
2270
2271           from += 1;
2272           n_left_from -= 1;
2273           to_next_drop[0] = pi0;
2274           to_next_drop += 1;
2275           n_left_to_next_drop -= 1;
2276
2277           p0->error = node->errors[drop0 ? IP4_ARP_ERROR_DROP : IP4_ARP_ERROR_REQUEST_SENT];
2278
2279           if (drop0)
2280             continue;
2281
2282           /*
2283            * Can happen if the control-plane is programming tables
2284            * with traffic flowing; at least that's today's lame excuse.
2285            */
2286           if (adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP)
2287             {
2288               p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
2289             }
2290           else
2291           /* Send ARP request. */
2292           {
2293             u32 bi0 = 0;
2294             vlib_buffer_t * b0;
2295             ethernet_arp_header_t * h0;
2296             vnet_hw_interface_t * hw_if0;
2297
2298             h0 = vlib_packet_template_get_packet (vm, &im->ip4_arp_request_packet_template, &bi0);
2299
2300             /* Add rewrite/encap string for ARP packet. */
2301             vnet_rewrite_one_header (adj0[0], h0, sizeof (ethernet_header_t));
2302
2303             hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2304
2305             /* Src ethernet address in ARP header. */
2306             clib_memcpy (h0->ip4_over_ethernet[0].ethernet, hw_if0->hw_address,
2307                     sizeof (h0->ip4_over_ethernet[0].ethernet));
2308
2309             ip4_src_address_for_packet (im, p0, &h0->ip4_over_ethernet[0].ip4, sw_if_index0);
2310
2311             /* Copy in destination address we are requesting. */
2312             h0->ip4_over_ethernet[1].ip4.data_u32 = ip0->dst_address.data_u32;
2313
2314             vlib_buffer_copy_trace_flag (vm, p0, bi0);
2315             b0 = vlib_get_buffer (vm, bi0);
2316             vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
2317
2318             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2319
2320             vlib_set_next_frame_buffer (vm, node, adj0->rewrite_header.next_index, bi0);
2321           }
2322         }
2323
2324       vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
2325     }
2326
2327   return frame->n_vectors;
2328 }
2329
2330 static char * ip4_arp_error_strings[] = {
2331   [IP4_ARP_ERROR_DROP] = "address overflow drops",
2332   [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
2333   [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
2334   [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
2335   [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
2336 };
2337
2338 VLIB_REGISTER_NODE (ip4_arp_node) = {
2339   .function = ip4_arp,
2340   .name = "ip4-arp",
2341   .vector_size = sizeof (u32),
2342
2343   .format_trace = format_ip4_forward_next_trace,
2344
2345   .n_errors = ARRAY_LEN (ip4_arp_error_strings),
2346   .error_strings = ip4_arp_error_strings,
2347
2348   .n_next_nodes = IP4_ARP_N_NEXT,
2349   .next_nodes = {
2350     [IP4_ARP_NEXT_DROP] = "error-drop",
2351   },
2352 };
2353
2354 #define foreach_notrace_ip4_arp_error           \
2355 _(DROP)                                         \
2356 _(REQUEST_SENT)                                 \
2357 _(REPLICATE_DROP)                               \
2358 _(REPLICATE_FAIL)
2359
2360 clib_error_t * arp_notrace_init (vlib_main_t * vm)
2361 {
2362   vlib_node_runtime_t *rt =
2363     vlib_node_get_runtime (vm, ip4_arp_node.index);
2364
2365   /* don't trace ARP request packets */
2366 #define _(a)                                    \
2367     vnet_pcap_drop_trace_filter_add_del         \
2368         (rt->errors[IP4_ARP_ERROR_##a],         \
2369          1 /* is_add */);
2370     foreach_notrace_ip4_arp_error;
2371 #undef _
2372   return 0;
2373 }
2374
2375 VLIB_INIT_FUNCTION(arp_notrace_init);
2376
2377
2378 /* Send an ARP request to see if given destination is reachable on given interface. */
2379 clib_error_t *
2380 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
2381 {
2382   vnet_main_t * vnm = vnet_get_main();
2383   ip4_main_t * im = &ip4_main;
2384   ethernet_arp_header_t * h;
2385   ip4_address_t * src;
2386   ip_interface_address_t * ia;
2387   ip_adjacency_t * adj;
2388   vnet_hw_interface_t * hi;
2389   vnet_sw_interface_t * si;
2390   vlib_buffer_t * b;
2391   u32 bi = 0;
2392
2393   si = vnet_get_sw_interface (vnm, sw_if_index);
2394
2395   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2396     {
2397       return clib_error_return (0, "%U: interface %U down",
2398                                 format_ip4_address, dst,
2399                                 format_vnet_sw_if_index_name, vnm,
2400                                 sw_if_index);
2401     }
2402
2403   src = ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2404   if (! src)
2405     {
2406       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2407       return clib_error_return
2408         (0, "no matching interface address for destination %U (interface %U)",
2409          format_ip4_address, dst,
2410          format_vnet_sw_if_index_name, vnm, sw_if_index);
2411     }
2412
2413   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
2414
2415   h = vlib_packet_template_get_packet (vm, &im->ip4_arp_request_packet_template, &bi);
2416
2417   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2418
2419   clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address, sizeof (h->ip4_over_ethernet[0].ethernet));
2420
2421   h->ip4_over_ethernet[0].ip4 = src[0];
2422   h->ip4_over_ethernet[1].ip4 = dst[0];
2423
2424   b = vlib_get_buffer (vm, bi);
2425   vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2426
2427   /* Add encapsulation string for software interface (e.g. ethernet header). */
2428   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2429   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2430
2431   {
2432     vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
2433     u32 * to_next = vlib_frame_vector_args (f);
2434     to_next[0] = bi;
2435     f->n_vectors = 1;
2436     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2437   }
2438
2439   return /* no error */ 0;
2440 }
2441
2442 typedef enum {
2443   IP4_REWRITE_NEXT_DROP,
2444   IP4_REWRITE_NEXT_ARP,
2445 } ip4_rewrite_next_t;
2446
2447 always_inline uword
2448 ip4_rewrite_inline (vlib_main_t * vm,
2449                     vlib_node_runtime_t * node,
2450                     vlib_frame_t * frame,
2451                     int rewrite_for_locally_received_packets)
2452 {
2453   ip_lookup_main_t * lm = &ip4_main.lookup_main;
2454   u32 * from = vlib_frame_vector_args (frame);
2455   u32 n_left_from, n_left_to_next, * to_next, next_index;
2456   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip4_input_node.index);
2457   vlib_rx_or_tx_t adj_rx_tx = rewrite_for_locally_received_packets ? VLIB_RX : VLIB_TX;
2458
2459   n_left_from = frame->n_vectors;
2460   next_index = node->cached_next_index;
2461   u32 cpu_index = os_get_cpu_number();
2462
2463   while (n_left_from > 0)
2464     {
2465       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2466
2467       while (n_left_from >= 4 && n_left_to_next >= 2)
2468         {
2469           ip_adjacency_t * adj0, * adj1;
2470           vlib_buffer_t * p0, * p1;
2471           ip4_header_t * ip0, * ip1;
2472           u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
2473           u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
2474           u32 next0_override, next1_override;
2475
2476           if (rewrite_for_locally_received_packets)
2477               next0_override = next1_override = 0;
2478
2479           /* Prefetch next iteration. */
2480           {
2481             vlib_buffer_t * p2, * p3;
2482
2483             p2 = vlib_get_buffer (vm, from[2]);
2484             p3 = vlib_get_buffer (vm, from[3]);
2485
2486             vlib_prefetch_buffer_header (p2, STORE);
2487             vlib_prefetch_buffer_header (p3, STORE);
2488
2489             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2490             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2491           }
2492
2493           pi0 = to_next[0] = from[0];
2494           pi1 = to_next[1] = from[1];
2495
2496           from += 2;
2497           n_left_from -= 2;
2498           to_next += 2;
2499           n_left_to_next -= 2;
2500
2501           p0 = vlib_get_buffer (vm, pi0);
2502           p1 = vlib_get_buffer (vm, pi1);
2503
2504           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2505           adj_index1 = vnet_buffer (p1)->ip.adj_index[adj_rx_tx];
2506
2507           /* We should never rewrite a pkt using the MISS adjacency */
2508           ASSERT(adj_index0 && adj_index1);
2509
2510           ip0 = vlib_buffer_get_current (p0);
2511           ip1 = vlib_buffer_get_current (p1);
2512
2513           error0 = error1 = IP4_ERROR_NONE;
2514
2515           /* Decrement TTL & update checksum.
2516              Works either endian, so no need for byte swap. */
2517           if (! rewrite_for_locally_received_packets)
2518             {
2519               i32 ttl0 = ip0->ttl, ttl1 = ip1->ttl;
2520
2521               /* Input node should have reject packets with ttl 0. */
2522               ASSERT (ip0->ttl > 0);
2523               ASSERT (ip1->ttl > 0);
2524
2525               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2526               checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2527
2528               checksum0 += checksum0 >= 0xffff;
2529               checksum1 += checksum1 >= 0xffff;
2530
2531               ip0->checksum = checksum0;
2532               ip1->checksum = checksum1;
2533
2534               ttl0 -= 1;
2535               ttl1 -= 1;
2536
2537               ip0->ttl = ttl0;
2538               ip1->ttl = ttl1;
2539
2540               error0 = ttl0 <= 0 ? IP4_ERROR_TIME_EXPIRED : error0;
2541               error1 = ttl1 <= 0 ? IP4_ERROR_TIME_EXPIRED : error1;
2542
2543               /* Verify checksum. */
2544               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2545               ASSERT (ip1->checksum == ip4_header_checksum (ip1));
2546             }
2547
2548           /* Rewrite packet header and updates lengths. */
2549           adj0 = ip_get_adjacency (lm, adj_index0);
2550           adj1 = ip_get_adjacency (lm, adj_index1);
2551
2552           if (rewrite_for_locally_received_packets)
2553             {
2554               /*
2555                * If someone sends e.g. an icmp4 w/ src = dst = interface addr,
2556                * we end up here with a local adjacency in hand
2557                * The local adj rewrite data is 0xfefe on purpose.
2558                * Bad engineer, no donut for you.
2559                */
2560               if (PREDICT_FALSE(adj0->lookup_next_index
2561                                 == IP_LOOKUP_NEXT_LOCAL))
2562                 error0 = IP4_ERROR_SPOOFED_LOCAL_PACKETS;
2563               if (PREDICT_FALSE(adj0->lookup_next_index
2564                                 == IP_LOOKUP_NEXT_ARP))
2565                 next0_override = IP4_REWRITE_NEXT_ARP;
2566               if (PREDICT_FALSE(adj1->lookup_next_index
2567                                 == IP_LOOKUP_NEXT_LOCAL))
2568                 error1 = IP4_ERROR_SPOOFED_LOCAL_PACKETS;
2569               if (PREDICT_FALSE(adj1->lookup_next_index
2570                                 == IP_LOOKUP_NEXT_ARP))
2571                 next1_override = IP4_REWRITE_NEXT_ARP;
2572             }
2573
2574           /* Worth pipelining. No guarantee that adj0,1 are hot... */
2575           rw_len0 = adj0[0].rewrite_header.data_bytes;
2576           rw_len1 = adj1[0].rewrite_header.data_bytes;
2577           next0 = (error0 == IP4_ERROR_NONE)
2578             ? adj0[0].rewrite_header.next_index : 0;
2579
2580           if (rewrite_for_locally_received_packets)
2581               next0 = next0 && next0_override ? next0_override : next0;
2582
2583           next1 = (error1 == IP4_ERROR_NONE)
2584             ? adj1[0].rewrite_header.next_index : 0;
2585
2586           if (rewrite_for_locally_received_packets)
2587               next1 = next1 && next1_override ? next1_override : next1;
2588
2589           /*
2590            * We've already accounted for an ethernet_header_t elsewhere
2591            */
2592           if (PREDICT_FALSE (rw_len0 > sizeof(ethernet_header_t)))
2593               vlib_increment_combined_counter
2594                   (&lm->adjacency_counters,
2595                    cpu_index, adj_index0,
2596                    /* packet increment */ 0,
2597                    /* byte increment */ rw_len0-sizeof(ethernet_header_t));
2598
2599           if (PREDICT_FALSE (rw_len1 > sizeof(ethernet_header_t)))
2600               vlib_increment_combined_counter
2601                   (&lm->adjacency_counters,
2602                    cpu_index, adj_index1,
2603                    /* packet increment */ 0,
2604                    /* byte increment */ rw_len1-sizeof(ethernet_header_t));
2605
2606           /* Check MTU of outgoing interface. */
2607           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2608                     ? IP4_ERROR_MTU_EXCEEDED
2609                     : error0);
2610           error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
2611                     ? IP4_ERROR_MTU_EXCEEDED
2612                     : error1);
2613
2614           p0->current_data -= rw_len0;
2615           p1->current_data -= rw_len1;
2616
2617           p0->current_length += rw_len0;
2618           p1->current_length += rw_len1;
2619
2620           vnet_buffer (p0)->sw_if_index[VLIB_TX] = adj0[0].rewrite_header.sw_if_index;
2621           vnet_buffer (p1)->sw_if_index[VLIB_TX] = adj1[0].rewrite_header.sw_if_index;
2622
2623           p0->error = error_node->errors[error0];
2624           p1->error = error_node->errors[error1];
2625
2626           /* Guess we are only writing on simple Ethernet header. */
2627           vnet_rewrite_two_headers (adj0[0], adj1[0],
2628                                     ip0, ip1,
2629                                     sizeof (ethernet_header_t));
2630
2631           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2632                                            to_next, n_left_to_next,
2633                                            pi0, pi1, next0, next1);
2634         }
2635
2636       while (n_left_from > 0 && n_left_to_next > 0)
2637         {
2638           ip_adjacency_t * adj0;
2639           vlib_buffer_t * p0;
2640           ip4_header_t * ip0;
2641           u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2642           u32 next0_override;
2643
2644           if (rewrite_for_locally_received_packets)
2645               next0_override = 0;
2646
2647           pi0 = to_next[0] = from[0];
2648
2649           p0 = vlib_get_buffer (vm, pi0);
2650
2651           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2652
2653           /* We should never rewrite a pkt using the MISS adjacency */
2654           ASSERT(adj_index0);
2655
2656           adj0 = ip_get_adjacency (lm, adj_index0);
2657
2658           ip0 = vlib_buffer_get_current (p0);
2659
2660           error0 = IP4_ERROR_NONE;
2661           next0 = 0;            /* drop on error */
2662
2663           /* Decrement TTL & update checksum. */
2664           if (! rewrite_for_locally_received_packets)
2665             {
2666               i32 ttl0 = ip0->ttl;
2667
2668               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2669
2670               checksum0 += checksum0 >= 0xffff;
2671
2672               ip0->checksum = checksum0;
2673
2674               ASSERT (ip0->ttl > 0);
2675
2676               ttl0 -= 1;
2677
2678               ip0->ttl = ttl0;
2679
2680               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2681
2682               error0 = ttl0 <= 0 ? IP4_ERROR_TIME_EXPIRED : error0;
2683             }
2684
2685           if (rewrite_for_locally_received_packets)
2686             {
2687               /*
2688                * If someone sends e.g. an icmp4 w/ src = dst = interface addr,
2689                * we end up here with a local adjacency in hand
2690                * The local adj rewrite data is 0xfefe on purpose.
2691                * Bad engineer, no donut for you.
2692                */
2693               if (PREDICT_FALSE(adj0->lookup_next_index
2694                                 == IP_LOOKUP_NEXT_LOCAL))
2695                 error0 = IP4_ERROR_SPOOFED_LOCAL_PACKETS;
2696               /*
2697                * We have to override the next_index in ARP adjacencies,
2698                * because they're set up for ip4-arp, not this node...
2699                */
2700               if (PREDICT_FALSE(adj0->lookup_next_index
2701                                 == IP_LOOKUP_NEXT_ARP))
2702                 next0_override = IP4_REWRITE_NEXT_ARP;
2703             }
2704
2705           /* Guess we are only writing on simple Ethernet header. */
2706           vnet_rewrite_one_header (adj0[0], ip0,
2707                                    sizeof (ethernet_header_t));
2708
2709           /* Update packet buffer attributes/set output interface. */
2710           rw_len0 = adj0[0].rewrite_header.data_bytes;
2711
2712           if (PREDICT_FALSE (rw_len0 > sizeof(ethernet_header_t)))
2713               vlib_increment_combined_counter
2714                   (&lm->adjacency_counters,
2715                    cpu_index, adj_index0,
2716                    /* packet increment */ 0,
2717                    /* byte increment */ rw_len0-sizeof(ethernet_header_t));
2718
2719           /* Check MTU of outgoing interface. */
2720           error0 = (vlib_buffer_length_in_chain (vm, p0)
2721                     > adj0[0].rewrite_header.max_l3_packet_bytes
2722                     ? IP4_ERROR_MTU_EXCEEDED
2723                     : error0);
2724
2725           p0->error = error_node->errors[error0];
2726           p0->current_data -= rw_len0;
2727           p0->current_length += rw_len0;
2728           vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2729             adj0[0].rewrite_header.sw_if_index;
2730
2731           next0 = (error0 == IP4_ERROR_NONE)
2732             ? adj0[0].rewrite_header.next_index : 0;
2733
2734           if (rewrite_for_locally_received_packets)
2735               next0 = next0 && next0_override ? next0_override : next0;
2736
2737           from += 1;
2738           n_left_from -= 1;
2739           to_next += 1;
2740           n_left_to_next -= 1;
2741
2742           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2743                                            to_next, n_left_to_next,
2744                                            pi0, next0);
2745         }
2746
2747       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2748     }
2749
2750   /* Need to do trace after rewrites to pick up new packet data. */
2751   if (node->flags & VLIB_NODE_FLAG_TRACE)
2752     ip4_forward_next_trace (vm, node, frame, adj_rx_tx);
2753
2754   return frame->n_vectors;
2755 }
2756
2757 static uword
2758 ip4_rewrite_transit (vlib_main_t * vm,
2759                      vlib_node_runtime_t * node,
2760                      vlib_frame_t * frame)
2761 {
2762   return ip4_rewrite_inline (vm, node, frame,
2763                              /* rewrite_for_locally_received_packets */ 0);
2764 }
2765
2766 static uword
2767 ip4_rewrite_local (vlib_main_t * vm,
2768                    vlib_node_runtime_t * node,
2769                    vlib_frame_t * frame)
2770 {
2771   return ip4_rewrite_inline (vm, node, frame,
2772                              /* rewrite_for_locally_received_packets */ 1);
2773 }
2774
2775 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2776   .function = ip4_rewrite_transit,
2777   .name = "ip4-rewrite-transit",
2778   .vector_size = sizeof (u32),
2779
2780   .format_trace = format_ip4_forward_next_trace,
2781
2782   .n_next_nodes = 2,
2783   .next_nodes = {
2784     [IP4_REWRITE_NEXT_DROP] = "error-drop",
2785     [IP4_REWRITE_NEXT_ARP] = "ip4-arp",
2786   },
2787 };
2788
2789 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite_transit)
2790
2791 VLIB_REGISTER_NODE (ip4_rewrite_local_node) = {
2792   .function = ip4_rewrite_local,
2793   .name = "ip4-rewrite-local",
2794   .vector_size = sizeof (u32),
2795
2796   .sibling_of = "ip4-rewrite-transit",
2797
2798   .format_trace = format_ip4_forward_next_trace,
2799
2800   .n_next_nodes = 2,
2801   .next_nodes = {
2802     [IP4_REWRITE_NEXT_DROP] = "error-drop",
2803     [IP4_REWRITE_NEXT_ARP] = "ip4-arp",
2804   },
2805 };
2806
2807 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_local_node, ip4_rewrite_local)
2808
2809 static clib_error_t *
2810 add_del_interface_table (vlib_main_t * vm,
2811                          unformat_input_t * input,
2812                          vlib_cli_command_t * cmd)
2813 {
2814   vnet_main_t * vnm = vnet_get_main();
2815   clib_error_t * error = 0;
2816   u32 sw_if_index, table_id;
2817
2818   sw_if_index = ~0;
2819
2820   if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2821     {
2822       error = clib_error_return (0, "unknown interface `%U'",
2823                                  format_unformat_error, input);
2824       goto done;
2825     }
2826
2827   if (unformat (input, "%d", &table_id))
2828     ;
2829   else
2830     {
2831       error = clib_error_return (0, "expected table id `%U'",
2832                                  format_unformat_error, input);
2833       goto done;
2834     }
2835
2836   {
2837     ip4_main_t * im = &ip4_main;
2838     ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_id, IP4_ROUTE_FLAG_TABLE_ID);
2839
2840     if (fib)
2841       {
2842         vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
2843         im->fib_index_by_sw_if_index[sw_if_index] = fib->index;
2844     }
2845   }
2846
2847  done:
2848   return error;
2849 }
2850
2851 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) = {
2852   .path = "set interface ip table",
2853   .function = add_del_interface_table,
2854   .short_help = "Add/delete FIB table id for interface",
2855 };
2856
2857
2858 static uword
2859 ip4_lookup_multicast (vlib_main_t * vm,
2860                       vlib_node_runtime_t * node,
2861                       vlib_frame_t * frame)
2862 {
2863   ip4_main_t * im = &ip4_main;
2864   ip_lookup_main_t * lm = &im->lookup_main;
2865   vlib_combined_counter_main_t * cm = &im->lookup_main.adjacency_counters;
2866   u32 n_left_from, n_left_to_next, * from, * to_next;
2867   ip_lookup_next_t next;
2868   u32 cpu_index = os_get_cpu_number();
2869
2870   from = vlib_frame_vector_args (frame);
2871   n_left_from = frame->n_vectors;
2872   next = node->cached_next_index;
2873
2874   if (node->flags & VLIB_NODE_FLAG_TRACE)
2875     ip4_forward_next_trace(vm, node, frame, VLIB_TX);
2876
2877   while (n_left_from > 0)
2878     {
2879       vlib_get_next_frame (vm, node, next,
2880                            to_next, n_left_to_next);
2881
2882       while (n_left_from >= 4 && n_left_to_next >= 2)
2883         {
2884           vlib_buffer_t * p0, * p1;
2885           u32 pi0, pi1, adj_index0, adj_index1, wrong_next;
2886           ip_lookup_next_t next0, next1;
2887           ip4_header_t * ip0, * ip1;
2888           ip_adjacency_t * adj0, * adj1;
2889           u32 fib_index0, fib_index1;
2890           u32 flow_hash_config0, flow_hash_config1;
2891
2892           /* Prefetch next iteration. */
2893           {
2894             vlib_buffer_t * p2, * p3;
2895
2896             p2 = vlib_get_buffer (vm, from[2]);
2897             p3 = vlib_get_buffer (vm, from[3]);
2898
2899             vlib_prefetch_buffer_header (p2, LOAD);
2900             vlib_prefetch_buffer_header (p3, LOAD);
2901
2902             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
2903             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
2904           }
2905
2906           pi0 = to_next[0] = from[0];
2907           pi1 = to_next[1] = from[1];
2908
2909           p0 = vlib_get_buffer (vm, pi0);
2910           p1 = vlib_get_buffer (vm, pi1);
2911
2912           ip0 = vlib_buffer_get_current (p0);
2913           ip1 = vlib_buffer_get_current (p1);
2914
2915           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
2916           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
2917           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
2918             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
2919           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
2920             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
2921
2922           adj_index0 = ip4_fib_lookup_buffer (im, fib_index0,
2923                                               &ip0->dst_address, p0);
2924           adj_index1 = ip4_fib_lookup_buffer (im, fib_index1,
2925                                               &ip1->dst_address, p1);
2926
2927           adj0 = ip_get_adjacency (lm, adj_index0);
2928           adj1 = ip_get_adjacency (lm, adj_index1);
2929
2930           next0 = adj0->lookup_next_index;
2931           next1 = adj1->lookup_next_index;
2932
2933           flow_hash_config0 =
2934               vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
2935
2936           flow_hash_config1 =
2937               vec_elt_at_index (im->fibs, fib_index1)->flow_hash_config;
2938
2939           vnet_buffer (p0)->ip.flow_hash = ip4_compute_flow_hash
2940               (ip0, flow_hash_config0);
2941
2942           vnet_buffer (p1)->ip.flow_hash = ip4_compute_flow_hash
2943               (ip1, flow_hash_config1);
2944
2945           ASSERT (adj0->n_adj > 0);
2946           ASSERT (adj1->n_adj > 0);
2947           ASSERT (is_pow2 (adj0->n_adj));
2948           ASSERT (is_pow2 (adj1->n_adj));
2949           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
2950           adj_index1 += (vnet_buffer (p1)->ip.flow_hash & (adj1->n_adj - 1));
2951
2952           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
2953           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
2954
2955           if (1) /* $$$$$$ HACK FIXME */
2956           vlib_increment_combined_counter
2957               (cm, cpu_index, adj_index0, 1,
2958                vlib_buffer_length_in_chain (vm, p0));
2959           if (1) /* $$$$$$ HACK FIXME */
2960           vlib_increment_combined_counter
2961               (cm, cpu_index, adj_index1, 1,
2962                vlib_buffer_length_in_chain (vm, p1));
2963
2964           from += 2;
2965           to_next += 2;
2966           n_left_to_next -= 2;
2967           n_left_from -= 2;
2968
2969           wrong_next = (next0 != next) + 2*(next1 != next);
2970           if (PREDICT_FALSE (wrong_next != 0))
2971             {
2972               switch (wrong_next)
2973                 {
2974                 case 1:
2975                   /* A B A */
2976                   to_next[-2] = pi1;
2977                   to_next -= 1;
2978                   n_left_to_next += 1;
2979                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
2980                   break;
2981
2982                 case 2:
2983                   /* A A B */
2984                   to_next -= 1;
2985                   n_left_to_next += 1;
2986                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
2987                   break;
2988
2989                 case 3:
2990                   /* A B C */
2991                   to_next -= 2;
2992                   n_left_to_next += 2;
2993                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
2994                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
2995                   if (next0 == next1)
2996                     {
2997                       /* A B B */
2998                       vlib_put_next_frame (vm, node, next, n_left_to_next);
2999                       next = next1;
3000                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
3001                     }
3002                 }
3003             }
3004         }
3005
3006       while (n_left_from > 0 && n_left_to_next > 0)
3007         {
3008           vlib_buffer_t * p0;
3009           ip4_header_t * ip0;
3010           u32 pi0, adj_index0;
3011           ip_lookup_next_t next0;
3012           ip_adjacency_t * adj0;
3013           u32 fib_index0;
3014           u32 flow_hash_config0;
3015
3016           pi0 = from[0];
3017           to_next[0] = pi0;
3018
3019           p0 = vlib_get_buffer (vm, pi0);
3020
3021           ip0 = vlib_buffer_get_current (p0);
3022
3023           fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
3024                                 vnet_buffer (p0)->sw_if_index[VLIB_RX]);
3025           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
3026               fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
3027
3028           adj_index0 = ip4_fib_lookup_buffer (im, fib_index0,
3029                                               &ip0->dst_address, p0);
3030
3031           adj0 = ip_get_adjacency (lm, adj_index0);
3032
3033           next0 = adj0->lookup_next_index;
3034
3035           flow_hash_config0 =
3036               vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
3037
3038           vnet_buffer (p0)->ip.flow_hash =
3039             ip4_compute_flow_hash (ip0, flow_hash_config0);
3040
3041           ASSERT (adj0->n_adj > 0);
3042           ASSERT (is_pow2 (adj0->n_adj));
3043           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
3044
3045           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
3046
3047           if (1) /* $$$$$$ HACK FIXME */
3048               vlib_increment_combined_counter
3049                   (cm, cpu_index, adj_index0, 1,
3050                    vlib_buffer_length_in_chain (vm, p0));
3051
3052           from += 1;
3053           to_next += 1;
3054           n_left_to_next -= 1;
3055           n_left_from -= 1;
3056
3057           if (PREDICT_FALSE (next0 != next))
3058             {
3059               n_left_to_next += 1;
3060               vlib_put_next_frame (vm, node, next, n_left_to_next);
3061               next = next0;
3062               vlib_get_next_frame (vm, node, next,
3063                                    to_next, n_left_to_next);
3064               to_next[0] = pi0;
3065               to_next += 1;
3066               n_left_to_next -= 1;
3067             }
3068         }
3069
3070       vlib_put_next_frame (vm, node, next, n_left_to_next);
3071     }
3072
3073   return frame->n_vectors;
3074 }
3075
3076 VLIB_REGISTER_NODE (ip4_lookup_multicast_node,static) = {
3077   .function = ip4_lookup_multicast,
3078   .name = "ip4-lookup-multicast",
3079   .vector_size = sizeof (u32),
3080
3081   .format_trace = format_ip4_forward_next_trace,
3082
3083   .n_next_nodes = IP_LOOKUP_N_NEXT,
3084   .next_nodes = IP4_LOOKUP_NEXT_NODES,
3085 };
3086
3087 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_multicast_node, ip4_lookup_multicast)
3088
3089 VLIB_REGISTER_NODE (ip4_multicast_node,static) = {
3090   .function = ip4_drop,
3091   .name = "ip4-multicast",
3092   .vector_size = sizeof (u32),
3093
3094   .format_trace = format_ip4_forward_next_trace,
3095
3096   .n_next_nodes = 1,
3097   .next_nodes = {
3098     [0] = "error-drop",
3099   },
3100 };
3101
3102 int ip4_lookup_validate (ip4_address_t *a, u32 fib_index0)
3103 {
3104   ip4_main_t * im = &ip4_main;
3105   ip4_fib_mtrie_t * mtrie0;
3106   ip4_fib_mtrie_leaf_t leaf0;
3107   u32 adj_index0;
3108
3109   mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
3110
3111   leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
3112   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 0);
3113   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 1);
3114   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
3115   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
3116
3117   /* Handle default route. */
3118   leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
3119
3120   adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
3121
3122   return adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
3123                                                   a,
3124                                                   /* no_default_route */ 0);
3125 }
3126
3127 static clib_error_t *
3128 test_lookup_command_fn (vlib_main_t * vm,
3129                         unformat_input_t * input,
3130                         vlib_cli_command_t * cmd)
3131 {
3132   u32 table_id = 0;
3133   f64 count = 1;
3134   u32 n;
3135   int i;
3136   ip4_address_t ip4_base_address;
3137   u64 errors = 0;
3138
3139   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3140       if (unformat (input, "table %d", &table_id))
3141         ;
3142       else if (unformat (input, "count %f", &count))
3143         ;
3144
3145       else if (unformat (input, "%U",
3146                          unformat_ip4_address, &ip4_base_address))
3147         ;
3148       else
3149         return clib_error_return (0, "unknown input `%U'",
3150                                   format_unformat_error, input);
3151   }
3152
3153   n = count;
3154
3155   for (i = 0; i < n; i++)
3156     {
3157       if (!ip4_lookup_validate (&ip4_base_address, table_id))
3158         errors++;
3159
3160       ip4_base_address.as_u32 =
3161         clib_host_to_net_u32 (1 +
3162                               clib_net_to_host_u32 (ip4_base_address.as_u32));
3163     }
3164
3165   if (errors)
3166     vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
3167   else
3168     vlib_cli_output (vm, "No errors in %d lookups\n", n);
3169
3170   return 0;
3171 }
3172
3173 VLIB_CLI_COMMAND (lookup_test_command, static) = {
3174     .path = "test lookup",
3175     .short_help = "test lookup",
3176     .function = test_lookup_command_fn,
3177 };
3178
3179 int vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
3180 {
3181   ip4_main_t * im4 = &ip4_main;
3182   ip4_fib_t * fib;
3183   uword * p = hash_get (im4->fib_index_by_table_id, table_id);
3184
3185   if (p == 0)
3186     return VNET_API_ERROR_NO_SUCH_FIB;
3187
3188   fib = vec_elt_at_index (im4->fibs, p[0]);
3189
3190   fib->flow_hash_config = flow_hash_config;
3191   return 0;
3192 }
3193
3194 static clib_error_t *
3195 set_ip_flow_hash_command_fn (vlib_main_t * vm,
3196                              unformat_input_t * input,
3197                              vlib_cli_command_t * cmd)
3198 {
3199   int matched = 0;
3200   u32 table_id = 0;
3201   u32 flow_hash_config = 0;
3202   int rv;
3203
3204   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3205     if (unformat (input, "table %d", &table_id))
3206       matched = 1;
3207 #define _(a,v) \
3208     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
3209     foreach_flow_hash_bit
3210 #undef _
3211     else break;
3212   }
3213
3214   if (matched == 0)
3215     return clib_error_return (0, "unknown input `%U'",
3216                               format_unformat_error, input);
3217
3218   rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
3219   switch (rv)
3220     {
3221     case 0:
3222       break;
3223
3224     case VNET_API_ERROR_NO_SUCH_FIB:
3225       return clib_error_return (0, "no such FIB table %d", table_id);
3226
3227     default:
3228       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3229       break;
3230     }
3231
3232   return 0;
3233 }
3234
3235 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) = {
3236   .path = "set ip flow-hash",
3237   .short_help =
3238   "set ip table flow-hash table <fib-id> src dst sport dport proto reverse",
3239   .function = set_ip_flow_hash_command_fn,
3240 };
3241
3242 int vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3243                                  u32 table_index)
3244 {
3245   vnet_main_t * vnm = vnet_get_main();
3246   vnet_interface_main_t * im = &vnm->interface_main;
3247   ip4_main_t * ipm = &ip4_main;
3248   ip_lookup_main_t * lm = &ipm->lookup_main;
3249   vnet_classify_main_t * cm = &vnet_classify_main;
3250
3251   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3252     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3253
3254   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3255     return VNET_API_ERROR_NO_SUCH_ENTRY;
3256
3257   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3258   lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
3259
3260   return 0;
3261 }
3262
3263 static clib_error_t *
3264 set_ip_classify_command_fn (vlib_main_t * vm,
3265                             unformat_input_t * input,
3266                             vlib_cli_command_t * cmd)
3267 {
3268   u32 table_index = ~0;
3269   int table_index_set = 0;
3270   u32 sw_if_index = ~0;
3271   int rv;
3272
3273   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3274     if (unformat (input, "table-index %d", &table_index))
3275       table_index_set = 1;
3276     else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3277                        vnet_get_main(), &sw_if_index))
3278       ;
3279     else
3280       break;
3281   }
3282
3283   if (table_index_set == 0)
3284     return clib_error_return (0, "classify table-index must be specified");
3285
3286   if (sw_if_index == ~0)
3287     return clib_error_return (0, "interface / subif must be specified");
3288
3289   rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
3290
3291   switch (rv)
3292     {
3293     case 0:
3294       break;
3295
3296     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3297       return clib_error_return (0, "No such interface");
3298
3299     case VNET_API_ERROR_NO_SUCH_ENTRY:
3300       return clib_error_return (0, "No such classifier table");
3301     }
3302   return 0;
3303 }
3304
3305 VLIB_CLI_COMMAND (set_ip_classify_command, static) = {
3306     .path = "set ip classify",
3307     .short_help =
3308     "set ip classify intfc <int> table-index <index>",
3309     .function = set_ip_classify_command_fn,
3310 };
3311