vnet/vnet/ip/ip4_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip4_forward.c: IP v4 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  43 #include <vnet/ethernet/arp_packet.h>   /* for ethernet_arp_header_t */
  44 #include <vnet/ppp/ppp.h>
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vnet/api_errno.h>     /* for API error numbers */
  47
  48 /* This is really, really simple but stupid fib. */
  49 u32
  50 ip4_fib_lookup_with_table (ip4_main_t * im, u32 fib_index,
  51                            ip4_address_t * dst,
  52                            u32 disable_default_route)
  53 {
  54   ip_lookup_main_t * lm = &im->lookup_main;
  55   ip4_fib_t * fib = vec_elt_at_index (im->fibs, fib_index);
  56   uword * p, * hash, key;
  57   i32 i, i_min, dst_address, ai;
  58
  59   i_min = disable_default_route ? 1 : 0;
  60   dst_address = clib_mem_unaligned (&dst->data_u32, u32);
  61   for (i = ARRAY_LEN (fib->adj_index_by_dst_address) - 1; i >= i_min; i--)
  62     {
  63       hash = fib->adj_index_by_dst_address[i];
  64       if (! hash)
  65         continue;
  66
  67       key = dst_address & im->fib_masks[i];
  68       if ((p = hash_get (hash, key)) != 0)
  69         {
  70           ai = p[0];
  71           goto done;
  72         }
  73     }
  74
  75   /* Nothing matches in table. */
  76   ai = lm->miss_adj_index;
  77
  78  done:
  79   return ai;
  80 }
  81
  82 static ip4_fib_t *
  83 create_fib_with_table_id (ip4_main_t * im, u32 table_id)
  84 {
  85   ip4_fib_t * fib;
  86   hash_set (im->fib_index_by_table_id, table_id, vec_len (im->fibs));
  87   vec_add2 (im->fibs, fib, 1);
  88   fib->table_id = table_id;
  89   fib->index = fib - im->fibs;
  90   fib->flow_hash_config = IP_FLOW_HASH_DEFAULT;
  91   fib->fwd_classify_table_index = ~0;
  92   fib->rev_classify_table_index = ~0;
  93   ip4_mtrie_init (&fib->mtrie);
  94   return fib;
  95 }
  96
  97 ip4_fib_t *
  98 find_ip4_fib_by_table_index_or_id (ip4_main_t * im,
  99                                    u32 table_index_or_id, u32 flags)
 100 {
 101   uword * p, fib_index;
 102
 103   fib_index = table_index_or_id;
 104   if (! (flags & IP4_ROUTE_FLAG_FIB_INDEX))
 105     {
 106       if (table_index_or_id == ~0) {
 107         table_index_or_id = 0;
 108         while ((p = hash_get (im->fib_index_by_table_id, table_index_or_id))) {
 109           table_index_or_id++;
 110         }
 111         return create_fib_with_table_id (im, table_index_or_id);
 112       }
 113
 114       p = hash_get (im->fib_index_by_table_id, table_index_or_id);
 115       if (! p)
 116         return create_fib_with_table_id (im, table_index_or_id);
 117       fib_index = p[0];
 118     }
 119   return vec_elt_at_index (im->fibs, fib_index);
 120 }
 121
 122 static void
 123 ip4_fib_init_adj_index_by_dst_address (ip_lookup_main_t * lm,
 124                                        ip4_fib_t * fib,
 125                                        u32 address_length)
 126 {
 127   hash_t * h;
 128   uword max_index;
 129
 130   ASSERT (lm->fib_result_n_bytes >= sizeof (uword));
 131   lm->fib_result_n_words = round_pow2 (lm->fib_result_n_bytes, sizeof (uword)) / sizeof (uword);
 132
 133   fib->adj_index_by_dst_address[address_length] =
 134     hash_create (32 /* elts */, lm->fib_result_n_words * sizeof (uword));
 135
 136   hash_set_flags (fib->adj_index_by_dst_address[address_length],
 137                   HASH_FLAG_NO_AUTO_SHRINK);
 138
 139   h = hash_header (fib->adj_index_by_dst_address[address_length]);
 140   max_index = (hash_value_bytes (h) / sizeof (fib->new_hash_values[0])) - 1;
 141
 142   /* Initialize new/old hash value vectors. */
 143   vec_validate_init_empty (fib->new_hash_values, max_index, ~0);
 144   vec_validate_init_empty (fib->old_hash_values, max_index, ~0);
 145 }
 146
 147 static void
 148 ip4_fib_set_adj_index (ip4_main_t * im,
 149                        ip4_fib_t * fib,
 150                        u32 flags,
 151                        u32 dst_address_u32,
 152                        u32 dst_address_length,
 153                        u32 adj_index)
 154 {
 155   ip_lookup_main_t * lm = &im->lookup_main;
 156   uword * hash;
 157
 158   if (vec_bytes(fib->old_hash_values))
 159     memset (fib->old_hash_values, ~0, vec_bytes (fib->old_hash_values));
 160   if (vec_bytes(fib->new_hash_values))
 161     memset (fib->new_hash_values, ~0, vec_bytes (fib->new_hash_values));
 162   fib->new_hash_values[0] = adj_index;
 163
 164   /* Make sure adj index is valid. */
 165   if (CLIB_DEBUG > 0)
 166     (void) ip_get_adjacency (lm, adj_index);
 167
 168   hash = fib->adj_index_by_dst_address[dst_address_length];
 169
 170   hash = _hash_set3 (hash, dst_address_u32,
 171                      fib->new_hash_values,
 172                      fib->old_hash_values);
 173
 174   fib->adj_index_by_dst_address[dst_address_length] = hash;
 175
 176   if (vec_len (im->add_del_route_callbacks) > 0)
 177     {
 178       ip4_add_del_route_callback_t * cb;
 179       ip4_address_t d;
 180       uword * p;
 181
 182       d.data_u32 = dst_address_u32;
 183       vec_foreach (cb, im->add_del_route_callbacks)
 184         if ((flags & cb->required_flags) == cb->required_flags)
 185           cb->function (im, cb->function_opaque,
 186                         fib, flags,
 187                         &d, dst_address_length,
 188                         fib->old_hash_values,
 189                         fib->new_hash_values);
 190
 191       p = hash_get (hash, dst_address_u32);
 192       clib_memcpy (p, fib->new_hash_values, vec_bytes (fib->new_hash_values));
 193     }
 194 }
 195
 196 void ip4_add_del_route (ip4_main_t * im, ip4_add_del_route_args_t * a)
 197 {
 198   ip_lookup_main_t * lm = &im->lookup_main;
 199   ip4_fib_t * fib;
 200   u32 dst_address, dst_address_length, adj_index, old_adj_index;
 201   uword * hash, is_del;
 202   ip4_add_del_route_callback_t * cb;
 203
 204   /* Either create new adjacency or use given one depending on arguments. */
 205   if (a->n_add_adj > 0)
 206     {
 207       ip_add_adjacency (lm, a->add_adj, a->n_add_adj, &adj_index);
 208       ip_call_add_del_adjacency_callbacks (lm, adj_index, /* is_del */ 0);
 209     }
 210   else
 211     adj_index = a->adj_index;
 212
 213   dst_address = a->dst_address.data_u32;
 214   dst_address_length = a->dst_address_length;
 215   fib = find_ip4_fib_by_table_index_or_id (im, a->table_index_or_table_id, a->flags);
 216
 217   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 218   dst_address &= im->fib_masks[dst_address_length];
 219
 220   if (! fib->adj_index_by_dst_address[dst_address_length])
 221     ip4_fib_init_adj_index_by_dst_address (lm, fib, dst_address_length);
 222
 223   hash = fib->adj_index_by_dst_address[dst_address_length];
 224
 225   is_del = (a->flags & IP4_ROUTE_FLAG_DEL) != 0;
 226
 227   if (is_del)
 228     {
 229       fib->old_hash_values[0] = ~0;
 230       hash = _hash_unset (hash, dst_address, fib->old_hash_values);
 231       fib->adj_index_by_dst_address[dst_address_length] = hash;
 232
 233       if (vec_len (im->add_del_route_callbacks) > 0
 234           && fib->old_hash_values[0] != ~0) /* make sure destination was found in hash */
 235         {
 236           fib->new_hash_values[0] = ~0;
 237           vec_foreach (cb, im->add_del_route_callbacks)
 238             if ((a->flags & cb->required_flags) == cb->required_flags)
 239               cb->function (im, cb->function_opaque,
 240                             fib, a->flags,
 241                             &a->dst_address, dst_address_length,
 242                             fib->old_hash_values,
 243                             fib->new_hash_values);
 244         }
 245     }
 246   else
 247     ip4_fib_set_adj_index (im, fib, a->flags, dst_address, dst_address_length,
 248                            adj_index);
 249
 250   old_adj_index = fib->old_hash_values[0];
 251
 252   /* Avoid spurious reference count increments */
 253   if (old_adj_index == adj_index
 254       && adj_index != ~0
 255       && !(a->flags & IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY))
 256     {
 257       ip_adjacency_t * adj = ip_get_adjacency (lm, adj_index);
 258       if (adj->share_count > 0)
 259         adj->share_count --;
 260     }
 261
 262   ip4_fib_mtrie_add_del_route (fib, a->dst_address, dst_address_length,
 263                                is_del ? old_adj_index : adj_index,
 264                                is_del);
 265
 266   /* Delete old adjacency index if present and changed. */
 267   if (! (a->flags & IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY)
 268       && old_adj_index != ~0
 269       && old_adj_index != adj_index)
 270     ip_del_adjacency (lm, old_adj_index);
 271 }
 272
 273 void
 274 ip4_add_del_route_next_hop (ip4_main_t * im,
 275                             u32 flags,
 276                             ip4_address_t * dst_address,
 277                             u32 dst_address_length,
 278                             ip4_address_t * next_hop,
 279                             u32 next_hop_sw_if_index,
 280                             u32 next_hop_weight, u32 adj_index,
 281                             u32 explicit_fib_index)
 282 {
 283   vnet_main_t * vnm = vnet_get_main();
 284   ip_lookup_main_t * lm = &im->lookup_main;
 285   u32 fib_index;
 286   ip4_fib_t * fib;
 287   u32 dst_address_u32, old_mp_adj_index, new_mp_adj_index;
 288   u32 dst_adj_index, nh_adj_index;
 289   uword * dst_hash, * dst_result;
 290   uword * nh_hash, * nh_result;
 291   ip_adjacency_t * dst_adj;
 292   ip_multipath_adjacency_t * old_mp, * new_mp;
 293   int is_del = (flags & IP4_ROUTE_FLAG_DEL) != 0;
 294   int is_interface_next_hop;
 295   clib_error_t * error = 0;
 296
 297   if (explicit_fib_index == (u32)~0)
 298       fib_index = vec_elt (im->fib_index_by_sw_if_index, next_hop_sw_if_index);
 299   else
 300       fib_index = explicit_fib_index;
 301
 302   fib = vec_elt_at_index (im->fibs, fib_index);
 303
 304   /* Lookup next hop to be added or deleted. */
 305   is_interface_next_hop = next_hop->data_u32 == 0;
 306   if (adj_index == (u32)~0)
 307     {
 308       if (is_interface_next_hop)
 309         {
 310           nh_result = hash_get (im->interface_route_adj_index_by_sw_if_index, next_hop_sw_if_index);
 311           if (nh_result)
 312             nh_adj_index = *nh_result;
 313           else
 314             {
 315               ip_adjacency_t * adj;
 316               adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 317                                       &nh_adj_index);
 318               ip4_adjacency_set_interface_route (vnm, adj, next_hop_sw_if_index, /* if_address_index */ ~0);
 319               ip_call_add_del_adjacency_callbacks (lm, nh_adj_index, /* is_del */ 0);
 320               hash_set (im->interface_route_adj_index_by_sw_if_index, next_hop_sw_if_index, nh_adj_index);
 321             }
 322         }
 323       else
 324         {
 325           nh_hash = fib->adj_index_by_dst_address[32];
 326           nh_result = hash_get (nh_hash, next_hop->data_u32);
 327
 328           /* Next hop must be known. */
 329           if (! nh_result)
 330             {
 331               ip_adjacency_t * adj;
 332
 333               nh_adj_index = ip4_fib_lookup_with_table (im, fib_index,
 334                                                         next_hop, 0);
 335               adj = ip_get_adjacency (lm, nh_adj_index);
 336               /* if ARP interface adjacencty is present, we need to
 337                  install ARP adjaceny for specific next hop */
 338               if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP &&
 339                   adj->arp.next_hop.ip4.as_u32 == 0)
 340                 {
 341                   nh_adj_index = vnet_arp_glean_add(fib_index, next_hop);
 342                 }
 343               else
 344                 {
 345                   /* Next hop is not known, so create indirect adj */
 346                   ip_adjacency_t add_adj;
 347                   memset (&add_adj, 0, sizeof(add_adj));
 348                   add_adj.n_adj = 1;
 349                   add_adj.lookup_next_index = IP_LOOKUP_NEXT_INDIRECT;
 350                   add_adj.indirect.next_hop.ip4.as_u32 = next_hop->as_u32;
 351                   add_adj.explicit_fib_index = explicit_fib_index;
 352                   ip_add_adjacency (lm, &add_adj, 1, &nh_adj_index);
 353                 }
 354             }
 355           else
 356             nh_adj_index = *nh_result;
 357         }
 358     }
 359   else
 360     {
 361       nh_adj_index = adj_index;
 362     }
 363   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 364   dst_address_u32 = dst_address->data_u32 & im->fib_masks[dst_address_length];
 365
 366   dst_hash = fib->adj_index_by_dst_address[dst_address_length];
 367   dst_result = hash_get (dst_hash, dst_address_u32);
 368   if (dst_result)
 369     {
 370       dst_adj_index = dst_result[0];
 371       dst_adj = ip_get_adjacency (lm, dst_adj_index);
 372     }
 373   else
 374     {
 375       /* For deletes destination must be known. */
 376       if (is_del)
 377         {
 378           vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
 379           error = clib_error_return (0, "unknown destination %U/%d",
 380                                      format_ip4_address, dst_address,
 381                                      dst_address_length);
 382           goto done;
 383         }
 384
 385       dst_adj_index = ~0;
 386       dst_adj = 0;
 387     }
 388
 389   /* Ignore adds of X/32 with next hop of X. */
 390   if (! is_del
 391       && dst_address_length == 32
 392       && dst_address->data_u32 == next_hop->data_u32
 393       && adj_index != (u32)~0)
 394     {
 395       vnm->api_errno = VNET_API_ERROR_PREFIX_MATCHES_NEXT_HOP;
 396       error = clib_error_return (0, "prefix matches next hop %U/%d",
 397                                  format_ip4_address, dst_address,
 398                                  dst_address_length);
 399       goto done;
 400     }
 401
 402   /* Destination is not known and default weight is set so add route
 403      to existing non-multipath adjacency */
 404   if (dst_adj_index == ~0 && next_hop_weight == 1 && next_hop_sw_if_index == ~0)
 405     {
 406       /* create new adjacency */
 407       ip4_add_del_route_args_t a;
 408       a.table_index_or_table_id = fib_index;
 409       a.flags = ((is_del ? IP4_ROUTE_FLAG_DEL : IP4_ROUTE_FLAG_ADD)
 410                  | IP4_ROUTE_FLAG_FIB_INDEX
 411                  | IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 412                  | (flags & (IP4_ROUTE_FLAG_NO_REDISTRIBUTE
 413                              | IP4_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
 414       a.dst_address = dst_address[0];
 415       a.dst_address_length = dst_address_length;
 416       a.adj_index = nh_adj_index;
 417       a.add_adj = 0;
 418       a.n_add_adj = 0;
 419
 420       ip4_add_del_route (im, &a);
 421
 422       goto done;
 423     }
 424
 425   old_mp_adj_index = dst_adj ? dst_adj->heap_handle : ~0;
 426
 427   if (! ip_multipath_adjacency_add_del_next_hop
 428       (lm, is_del,
 429        old_mp_adj_index,
 430        nh_adj_index,
 431        next_hop_weight,
 432        &new_mp_adj_index))
 433     {
 434       vnm->api_errno = VNET_API_ERROR_NEXT_HOP_NOT_FOUND_MP;
 435       error = clib_error_return (0, "requested deleting next-hop %U not found in multi-path",
 436                                  format_ip4_address, next_hop);
 437       goto done;
 438     }
 439
 440   old_mp = new_mp = 0;
 441   if (old_mp_adj_index != ~0)
 442     old_mp = vec_elt_at_index (lm->multipath_adjacencies, old_mp_adj_index);
 443   if (new_mp_adj_index != ~0)
 444     new_mp = vec_elt_at_index (lm->multipath_adjacencies, new_mp_adj_index);
 445
 446   if (old_mp != new_mp)
 447     {
 448       ip4_add_del_route_args_t a;
 449       a.table_index_or_table_id = fib_index;
 450       a.flags = ((is_del && ! new_mp ? IP4_ROUTE_FLAG_DEL : IP4_ROUTE_FLAG_ADD)
 451                  | IP4_ROUTE_FLAG_FIB_INDEX
 452                  | IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 453                  | (flags & (IP4_ROUTE_FLAG_NO_REDISTRIBUTE | IP4_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
 454       a.dst_address = dst_address[0];
 455       a.dst_address_length = dst_address_length;
 456       a.adj_index = new_mp ? new_mp->adj_index : dst_adj_index;
 457       a.add_adj = 0;
 458       a.n_add_adj = 0;
 459
 460       ip4_add_del_route (im, &a);
 461     }
 462
 463  done:
 464   if (error)
 465     clib_error_report (error);
 466 }
 467
 468 void *
 469 ip4_get_route (ip4_main_t * im,
 470                u32 table_index_or_table_id,
 471                u32 flags,
 472                u8 * address,
 473                u32 address_length)
 474 {
 475   ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 476   u32 dst_address = * (u32 *) address;
 477   uword * hash, * p;
 478
 479   ASSERT (address_length < ARRAY_LEN (im->fib_masks));
 480   dst_address &= im->fib_masks[address_length];
 481
 482   hash = fib->adj_index_by_dst_address[address_length];
 483   p = hash_get (hash, dst_address);
 484   return (void *) p;
 485 }
 486
 487 void
 488 ip4_foreach_matching_route (ip4_main_t * im,
 489                             u32 table_index_or_table_id,
 490                             u32 flags,
 491                             ip4_address_t * address,
 492                             u32 address_length,
 493                             ip4_address_t ** results,
 494                             u8 ** result_lengths)
 495 {
 496   ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 497   u32 dst_address = address->data_u32;
 498   u32 this_length = address_length;
 499
 500   if (*results)
 501     _vec_len (*results) = 0;
 502   if (*result_lengths)
 503     _vec_len (*result_lengths) = 0;
 504
 505   while (this_length <= 32 && vec_len (results) == 0)
 506     {
 507       uword k, v;
 508       hash_foreach (k, v, fib->adj_index_by_dst_address[this_length], ({
 509         if (0 == ((k ^ dst_address) & im->fib_masks[address_length]))
 510           {
 511             ip4_address_t a;
 512             a.data_u32 = k;
 513             vec_add1 (*results, a);
 514             vec_add1 (*result_lengths, this_length);
 515           }
 516       }));
 517
 518       this_length++;
 519     }
 520 }
 521
 522 void ip4_maybe_remap_adjacencies (ip4_main_t * im,
 523                                   u32 table_index_or_table_id,
 524                                   u32 flags)
 525 {
 526   ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 527   ip_lookup_main_t * lm = &im->lookup_main;
 528   u32 i, l;
 529   ip4_address_t a;
 530   ip4_add_del_route_callback_t * cb;
 531   static ip4_address_t * to_delete;
 532
 533   if (lm->n_adjacency_remaps == 0)
 534     return;
 535
 536   for (l = 0; l <= 32; l++)
 537     {
 538       hash_pair_t * p;
 539       uword * hash = fib->adj_index_by_dst_address[l];
 540
 541       if (hash_elts (hash) == 0)
 542         continue;
 543
 544       if (to_delete)
 545         _vec_len (to_delete) = 0;
 546
 547       hash_foreach_pair (p, hash, ({
 548         u32 adj_index = p->value[0];
 549         u32 m = vec_elt (lm->adjacency_remap_table, adj_index);
 550
 551         if (m)
 552           {
 553             /* Record destination address from hash key. */
 554             a.data_u32 = p->key;
 555
 556             /* New adjacency points to nothing: so delete prefix. */
 557             if (m == ~0)
 558               vec_add1 (to_delete, a);
 559             else
 560               {
 561                 /* Remap to new adjacency. */
 562                 clib_memcpy (fib->old_hash_values, p->value, vec_bytes (fib->old_hash_values));
 563
 564                 /* Set new adjacency value. */
 565                 fib->new_hash_values[0] = p->value[0] = m - 1;
 566
 567                 vec_foreach (cb, im->add_del_route_callbacks)
 568                   if ((flags & cb->required_flags) == cb->required_flags)
 569                     cb->function (im, cb->function_opaque,
 570                                   fib, flags | IP4_ROUTE_FLAG_ADD,
 571                                   &a, l,
 572                                   fib->old_hash_values,
 573                                   fib->new_hash_values);
 574               }
 575           }
 576       }));
 577
 578       fib->new_hash_values[0] = ~0;
 579       for (i = 0; i < vec_len (to_delete); i++)
 580         {
 581           hash = _hash_unset (hash, to_delete[i].data_u32, fib->old_hash_values);
 582           vec_foreach (cb, im->add_del_route_callbacks)
 583             if ((flags & cb->required_flags) == cb->required_flags)
 584               cb->function (im, cb->function_opaque,
 585                             fib, flags | IP4_ROUTE_FLAG_DEL,
 586                             &a, l,
 587                             fib->old_hash_values,
 588                             fib->new_hash_values);
 589         }
 590     }
 591
 592   /* Also remap adjacencies in mtrie. */
 593   ip4_mtrie_maybe_remap_adjacencies (lm, &fib->mtrie);
 594
 595   /* Reset mapping table. */
 596   vec_zero (lm->adjacency_remap_table);
 597
 598   /* All remaps have been performed. */
 599   lm->n_adjacency_remaps = 0;
 600 }
 601
 602 void ip4_delete_matching_routes (ip4_main_t * im,
 603                                  u32 table_index_or_table_id,
 604                                  u32 flags,
 605                                  ip4_address_t * address,
 606                                  u32 address_length)
 607 {
 608   static ip4_address_t * matching_addresses;
 609   static u8 * matching_address_lengths;
 610   u32 l, i;
 611   ip4_add_del_route_args_t a;
 612
 613   a.flags = IP4_ROUTE_FLAG_DEL | IP4_ROUTE_FLAG_NO_REDISTRIBUTE | flags;
 614   a.table_index_or_table_id = table_index_or_table_id;
 615   a.adj_index = ~0;
 616   a.add_adj = 0;
 617   a.n_add_adj = 0;
 618
 619   for (l = address_length + 1; l <= 32; l++)
 620     {
 621       ip4_foreach_matching_route (im, table_index_or_table_id, flags,
 622                                   address,
 623                                   l,
 624                                   &matching_addresses,
 625                                   &matching_address_lengths);
 626       for (i = 0; i < vec_len (matching_addresses); i++)
 627         {
 628           a.dst_address = matching_addresses[i];
 629           a.dst_address_length = matching_address_lengths[i];
 630           ip4_add_del_route (im, &a);
 631         }
 632     }
 633
 634   ip4_maybe_remap_adjacencies (im, table_index_or_table_id, flags);
 635 }
 636
 637 void
 638 ip4_forward_next_trace (vlib_main_t * vm,
 639                         vlib_node_runtime_t * node,
 640                         vlib_frame_t * frame,
 641                         vlib_rx_or_tx_t which_adj_index);
 642
 643 always_inline uword
 644 ip4_lookup_inline (vlib_main_t * vm,
 645                    vlib_node_runtime_t * node,
 646                    vlib_frame_t * frame,
 647                    int lookup_for_responses_to_locally_received_packets,
 648                    int is_indirect)
 649 {
 650   ip4_main_t * im = &ip4_main;
 651   ip_lookup_main_t * lm = &im->lookup_main;
 652   vlib_combined_counter_main_t * cm = &im->lookup_main.adjacency_counters;
 653   u32 n_left_from, n_left_to_next, * from, * to_next;
 654   ip_lookup_next_t next;
 655   u32 cpu_index = os_get_cpu_number();
 656
 657   from = vlib_frame_vector_args (frame);
 658   n_left_from = frame->n_vectors;
 659   next = node->cached_next_index;
 660
 661   while (n_left_from > 0)
 662     {
 663       vlib_get_next_frame (vm, node, next,
 664                            to_next, n_left_to_next);
 665
 666       while (n_left_from >= 4 && n_left_to_next >= 2)
 667         {
 668           vlib_buffer_t * p0, * p1;
 669           ip4_header_t * ip0, * ip1;
 670           __attribute__((unused)) tcp_header_t * tcp0, * tcp1;
 671           ip_lookup_next_t next0, next1;
 672           ip_adjacency_t * adj0, * adj1;
 673           ip4_fib_mtrie_t * mtrie0, * mtrie1;
 674           ip4_fib_mtrie_leaf_t leaf0, leaf1;
 675           ip4_address_t * dst_addr0, *dst_addr1;
 676           __attribute__((unused)) u32 pi0, fib_index0, adj_index0, is_tcp_udp0;
 677           __attribute__((unused)) u32 pi1, fib_index1, adj_index1, is_tcp_udp1;
 678           u32 flow_hash_config0, flow_hash_config1;
 679           u32 hash_c0, hash_c1;
 680           u32 wrong_next;
 681
 682           /* Prefetch next iteration. */
 683           {
 684             vlib_buffer_t * p2, * p3;
 685
 686             p2 = vlib_get_buffer (vm, from[2]);
 687             p3 = vlib_get_buffer (vm, from[3]);
 688
 689             vlib_prefetch_buffer_header (p2, LOAD);
 690             vlib_prefetch_buffer_header (p3, LOAD);
 691
 692             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
 693             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
 694           }
 695
 696           pi0 = to_next[0] = from[0];
 697           pi1 = to_next[1] = from[1];
 698
 699           p0 = vlib_get_buffer (vm, pi0);
 700           p1 = vlib_get_buffer (vm, pi1);
 701
 702           ip0 = vlib_buffer_get_current (p0);
 703           ip1 = vlib_buffer_get_current (p1);
 704
 705           if (is_indirect)
 706             {
 707               ip_adjacency_t * iadj0, * iadj1;
 708               iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
 709               iadj1 = ip_get_adjacency (lm, vnet_buffer(p1)->ip.adj_index[VLIB_TX]);
 710               dst_addr0 = &iadj0->indirect.next_hop.ip4;
 711               dst_addr1 = &iadj1->indirect.next_hop.ip4;
 712             }
 713           else
 714             {
 715               dst_addr0 = &ip0->dst_address;
 716               dst_addr1 = &ip1->dst_address;
 717             }
 718
 719           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 720           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 721           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 722             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 723           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
 724             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
 725
 726
 727           if (! lookup_for_responses_to_locally_received_packets)
 728             {
 729               mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
 730               mtrie1 = &vec_elt_at_index (im->fibs, fib_index1)->mtrie;
 731
 732               leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
 733
 734               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 0);
 735               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 0);
 736             }
 737
 738           tcp0 = (void *) (ip0 + 1);
 739           tcp1 = (void *) (ip1 + 1);
 740
 741           is_tcp_udp0 = (ip0->protocol == IP_PROTOCOL_TCP
 742                          || ip0->protocol == IP_PROTOCOL_UDP);
 743           is_tcp_udp1 = (ip1->protocol == IP_PROTOCOL_TCP
 744                          || ip1->protocol == IP_PROTOCOL_UDP);
 745
 746           if (! lookup_for_responses_to_locally_received_packets)
 747             {
 748               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
 749               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 1);
 750             }
 751
 752           if (! lookup_for_responses_to_locally_received_packets)
 753             {
 754               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 755               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
 756             }
 757
 758           if (! lookup_for_responses_to_locally_received_packets)
 759             {
 760               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 761               leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
 762             }
 763
 764           if (lookup_for_responses_to_locally_received_packets)
 765             {
 766               adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 767               adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
 768             }
 769           else
 770             {
 771               /* Handle default route. */
 772               leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 773               leaf1 = (leaf1 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie1->default_leaf : leaf1);
 774
 775               adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 776               adj_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
 777             }
 778
 779           ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
 780                                                            dst_addr0,
 781                                                            /* no_default_route */ 0));
 782           ASSERT (adj_index1 == ip4_fib_lookup_with_table (im, fib_index1,
 783                                                            dst_addr1,
 784                                                            /* no_default_route */ 0));
 785           adj0 = ip_get_adjacency (lm, adj_index0);
 786           adj1 = ip_get_adjacency (lm, adj_index1);
 787
 788           next0 = adj0->lookup_next_index;
 789           next1 = adj1->lookup_next_index;
 790
 791           /* Use flow hash to compute multipath adjacency. */
 792           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 793           hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
 794           if (PREDICT_FALSE (adj0->n_adj > 1))
 795             {
 796               flow_hash_config0 =
 797                 vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
 798               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 799                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 800             }
 801           if (PREDICT_FALSE(adj1->n_adj > 1))
 802             {
 803               flow_hash_config1 =
 804                 vec_elt_at_index (im->fibs, fib_index1)->flow_hash_config;
 805               hash_c1 = vnet_buffer (p1)->ip.flow_hash =
 806                 ip4_compute_flow_hash (ip1, flow_hash_config1);
 807             }
 808
 809           ASSERT (adj0->n_adj > 0);
 810           ASSERT (adj1->n_adj > 0);
 811           ASSERT (is_pow2 (adj0->n_adj));
 812           ASSERT (is_pow2 (adj1->n_adj));
 813           adj_index0 += (hash_c0 & (adj0->n_adj - 1));
 814           adj_index1 += (hash_c1 & (adj1->n_adj - 1));
 815
 816           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 817           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
 818
 819           vlib_increment_combined_counter
 820               (cm, cpu_index, adj_index0, 1,
 821                vlib_buffer_length_in_chain (vm, p0)
 822                + sizeof(ethernet_header_t));
 823           vlib_increment_combined_counter
 824               (cm, cpu_index, adj_index1, 1,
 825                vlib_buffer_length_in_chain (vm, p1)
 826                + sizeof(ethernet_header_t));
 827
 828           from += 2;
 829           to_next += 2;
 830           n_left_to_next -= 2;
 831           n_left_from -= 2;
 832
 833           wrong_next = (next0 != next) + 2*(next1 != next);
 834           if (PREDICT_FALSE (wrong_next != 0))
 835             {
 836               switch (wrong_next)
 837                 {
 838                 case 1:
 839                   /* A B A */
 840                   to_next[-2] = pi1;
 841                   to_next -= 1;
 842                   n_left_to_next += 1;
 843                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 844                   break;
 845
 846                 case 2:
 847                   /* A A B */
 848                   to_next -= 1;
 849                   n_left_to_next += 1;
 850                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 851                   break;
 852
 853                 case 3:
 854                   /* A B C */
 855                   to_next -= 2;
 856                   n_left_to_next += 2;
 857                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 858                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 859                   if (next0 == next1)
 860                     {
 861                       /* A B B */
 862                       vlib_put_next_frame (vm, node, next, n_left_to_next);
 863                       next = next1;
 864                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 865                     }
 866                 }
 867             }
 868         }
 869
 870       while (n_left_from > 0 && n_left_to_next > 0)
 871         {
 872           vlib_buffer_t * p0;
 873           ip4_header_t * ip0;
 874           __attribute__((unused)) tcp_header_t * tcp0;
 875           ip_lookup_next_t next0;
 876           ip_adjacency_t * adj0;
 877           ip4_fib_mtrie_t * mtrie0;
 878           ip4_fib_mtrie_leaf_t leaf0;
 879           ip4_address_t * dst_addr0;
 880           __attribute__((unused)) u32 pi0, fib_index0, adj_index0, is_tcp_udp0;
 881           u32 flow_hash_config0, hash_c0;
 882
 883           pi0 = from[0];
 884           to_next[0] = pi0;
 885
 886           p0 = vlib_get_buffer (vm, pi0);
 887
 888           ip0 = vlib_buffer_get_current (p0);
 889
 890           if (is_indirect)
 891             {
 892               ip_adjacency_t * iadj0;
 893               iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
 894               dst_addr0 = &iadj0->indirect.next_hop.ip4;
 895             }
 896           else
 897             {
 898               dst_addr0 = &ip0->dst_address;
 899             }
 900
 901           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 902           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 903             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 904
 905           if (! lookup_for_responses_to_locally_received_packets)
 906             {
 907               mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
 908
 909               leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
 910
 911               leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 0);
 912             }
 913
 914           tcp0 = (void *) (ip0 + 1);
 915
 916           is_tcp_udp0 = (ip0->protocol == IP_PROTOCOL_TCP
 917                          || ip0->protocol == IP_PROTOCOL_UDP);
 918
 919           if (! lookup_for_responses_to_locally_received_packets)
 920             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
 921
 922           if (! lookup_for_responses_to_locally_received_packets)
 923             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
 924
 925           if (! lookup_for_responses_to_locally_received_packets)
 926             leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
 927
 928           if (lookup_for_responses_to_locally_received_packets)
 929             adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
 930           else
 931             {
 932               /* Handle default route. */
 933               leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
 934               adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
 935             }
 936
 937           ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
 938                                                            dst_addr0,
 939                                                            /* no_default_route */ 0));
 940
 941           adj0 = ip_get_adjacency (lm, adj_index0);
 942
 943           next0 = adj0->lookup_next_index;
 944
 945           /* Use flow hash to compute multipath adjacency. */
 946           hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
 947           if (PREDICT_FALSE(adj0->n_adj > 1))
 948             {
 949               flow_hash_config0 =
 950                 vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
 951
 952               hash_c0 = vnet_buffer (p0)->ip.flow_hash =
 953                 ip4_compute_flow_hash (ip0, flow_hash_config0);
 954             }
 955
 956           ASSERT (adj0->n_adj > 0);
 957           ASSERT (is_pow2 (adj0->n_adj));
 958           adj_index0 += (hash_c0 & (adj0->n_adj - 1));
 959
 960           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 961
 962           vlib_increment_combined_counter
 963               (cm, cpu_index, adj_index0, 1,
 964                vlib_buffer_length_in_chain (vm, p0)
 965                + sizeof(ethernet_header_t));
 966
 967           from += 1;
 968           to_next += 1;
 969           n_left_to_next -= 1;
 970           n_left_from -= 1;
 971
 972           if (PREDICT_FALSE (next0 != next))
 973             {
 974               n_left_to_next += 1;
 975               vlib_put_next_frame (vm, node, next, n_left_to_next);
 976               next = next0;
 977               vlib_get_next_frame (vm, node, next,
 978                                    to_next, n_left_to_next);
 979               to_next[0] = pi0;
 980               to_next += 1;
 981               n_left_to_next -= 1;
 982             }
 983         }
 984
 985       vlib_put_next_frame (vm, node, next, n_left_to_next);
 986     }
 987
 988   if (node->flags & VLIB_NODE_FLAG_TRACE)
 989     ip4_forward_next_trace(vm, node, frame, VLIB_TX);
 990
 991   return frame->n_vectors;
 992 }
 993
 994 static uword
 995 ip4_lookup (vlib_main_t * vm,
 996             vlib_node_runtime_t * node,
 997             vlib_frame_t * frame)
 998 {
 999   return ip4_lookup_inline (vm, node, frame,
1000                             /* lookup_for_responses_to_locally_received_packets */ 0,
1001                             /* is_indirect */ 0);
1002
1003 }
1004
1005 void ip4_adjacency_set_interface_route (vnet_main_t * vnm,
1006                                         ip_adjacency_t * adj,
1007                                         u32 sw_if_index,
1008                                         u32 if_address_index)
1009 {
1010   vnet_hw_interface_t * hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
1011   ip_lookup_next_t n;
1012   vnet_l3_packet_type_t packet_type;
1013   u32 node_index;
1014
1015   if (hw->hw_class_index == ethernet_hw_interface_class.index
1016       || hw->hw_class_index == srp_hw_interface_class.index)
1017     {
1018       /*
1019        * We have a bit of a problem in this case. ip4-arp uses
1020        * the rewrite_header.next_index to hand pkts to the
1021        * indicated inteface output node. We can end up in
1022        * ip4_rewrite_local, too, which also pays attention to
1023        * rewrite_header.next index. Net result: a hack in
1024        * ip4_rewrite_local...
1025        */
1026       n = IP_LOOKUP_NEXT_ARP;
1027       node_index = ip4_arp_node.index;
1028       adj->if_address_index = if_address_index;
1029       adj->arp.next_hop.ip4.as_u32 = 0;
1030       ip46_address_reset(&adj->arp.next_hop);
1031       packet_type = VNET_L3_PACKET_TYPE_ARP;
1032     }
1033   else
1034     {
1035       n = IP_LOOKUP_NEXT_REWRITE;
1036       node_index = ip4_rewrite_node.index;
1037       packet_type = VNET_L3_PACKET_TYPE_IP4;
1038     }
1039
1040   adj->lookup_next_index = n;
1041   vnet_rewrite_for_sw_interface
1042     (vnm,
1043      packet_type,
1044      sw_if_index,
1045      node_index,
1046      VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST,
1047      &adj->rewrite_header,
1048      sizeof (adj->rewrite_data));
1049 }
1050
1051 static void
1052 ip4_add_interface_routes (u32 sw_if_index,
1053                           ip4_main_t * im, u32 fib_index,
1054                           ip_interface_address_t * a)
1055 {
1056   vnet_main_t * vnm = vnet_get_main();
1057   ip_lookup_main_t * lm = &im->lookup_main;
1058   ip_adjacency_t * adj;
1059   ip4_address_t * address = ip_interface_address_get_address (lm, a);
1060   ip4_add_del_route_args_t x;
1061   vnet_hw_interface_t * hw_if = vnet_get_sup_hw_interface (vnm, sw_if_index);
1062   u32 classify_table_index;
1063
1064   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1065   x.table_index_or_table_id = fib_index;
1066   x.flags = (IP4_ROUTE_FLAG_ADD
1067              | IP4_ROUTE_FLAG_FIB_INDEX
1068              | IP4_ROUTE_FLAG_NO_REDISTRIBUTE);
1069   x.dst_address = address[0];
1070   x.dst_address_length = a->address_length;
1071   x.n_add_adj = 0;
1072   x.add_adj = 0;
1073
1074   a->neighbor_probe_adj_index = ~0;
1075   if (a->address_length < 32)
1076     {
1077       adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1078                               &x.adj_index);
1079       ip4_adjacency_set_interface_route (vnm, adj, sw_if_index, a - lm->if_address_pool);
1080       ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1081       ip4_add_del_route (im, &x);
1082       a->neighbor_probe_adj_index = x.adj_index;
1083     }
1084
1085   /* Add e.g. 1.1.1.1/32 as local to this host. */
1086   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1087                           &x.adj_index);
1088
1089   classify_table_index = ~0;
1090   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
1091     classify_table_index = lm->classify_table_index_by_sw_if_index [sw_if_index];
1092   if (classify_table_index != (u32) ~0)
1093     {
1094       adj->lookup_next_index = IP_LOOKUP_NEXT_CLASSIFY;
1095       adj->classify.table_index = classify_table_index;
1096     }
1097   else
1098     adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
1099
1100   adj->if_address_index = a - lm->if_address_pool;
1101   adj->rewrite_header.sw_if_index = sw_if_index;
1102   adj->rewrite_header.max_l3_packet_bytes = hw_if->max_l3_packet_bytes[VLIB_RX];
1103   /*
1104    * Local adjs are never to be rewritten. Spoofed pkts w/ src = dst = local
1105    * fail an RPF-ish check, but still go thru the rewrite code...
1106    */
1107   adj->rewrite_header.data_bytes = 0;
1108
1109   ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1110   x.dst_address_length = 32;
1111   ip4_add_del_route (im, &x);
1112 }
1113
1114 static void
1115 ip4_del_interface_routes (ip4_main_t * im, u32 fib_index, ip4_address_t * address, u32 address_length)
1116 {
1117   ip4_add_del_route_args_t x;
1118
1119   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1120   x.table_index_or_table_id = fib_index;
1121   x.flags = (IP4_ROUTE_FLAG_DEL
1122              | IP4_ROUTE_FLAG_FIB_INDEX
1123              | IP4_ROUTE_FLAG_NO_REDISTRIBUTE);
1124   x.dst_address = address[0];
1125   x.dst_address_length = address_length;
1126   x.adj_index = ~0;
1127   x.n_add_adj = 0;
1128   x.add_adj = 0;
1129
1130   if (address_length < 32)
1131     ip4_add_del_route (im, &x);
1132
1133   x.dst_address_length = 32;
1134   ip4_add_del_route (im, &x);
1135
1136   ip4_delete_matching_routes (im,
1137                               fib_index,
1138                               IP4_ROUTE_FLAG_FIB_INDEX,
1139                               address,
1140                               address_length);
1141 }
1142
1143 typedef struct {
1144     u32 sw_if_index;
1145     ip4_address_t address;
1146     u32 length;
1147 } ip4_interface_address_t;
1148
1149 static clib_error_t *
1150 ip4_add_del_interface_address_internal (vlib_main_t * vm,
1151                                         u32 sw_if_index,
1152                                         ip4_address_t * new_address,
1153                                         u32 new_length,
1154                                         u32 redistribute,
1155                                         u32 insert_routes,
1156                                         u32 is_del);
1157
1158 static clib_error_t *
1159 ip4_add_del_interface_address_internal (vlib_main_t * vm,
1160                                         u32 sw_if_index,
1161                                         ip4_address_t * address,
1162                                         u32 address_length,
1163                                         u32 redistribute,
1164                                         u32 insert_routes,
1165                                         u32 is_del)
1166 {
1167   vnet_main_t * vnm = vnet_get_main();
1168   ip4_main_t * im = &ip4_main;
1169   ip_lookup_main_t * lm = &im->lookup_main;
1170   clib_error_t * error = 0;
1171   u32 if_address_index, elts_before;
1172   ip4_address_fib_t ip4_af, * addr_fib = 0;
1173
1174   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1175   ip4_addr_fib_init (&ip4_af, address,
1176                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
1177   vec_add1 (addr_fib, ip4_af);
1178
1179   /* When adding an address check that it does not conflict with an existing address. */
1180   if (! is_del)
1181     {
1182       ip_interface_address_t * ia;
1183       foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
1184                                     0 /* honor unnumbered */,
1185       ({
1186         ip4_address_t * x = ip_interface_address_get_address (&im->lookup_main, ia);
1187
1188         if (ip4_destination_matches_route (im, address, x, ia->address_length)
1189             || ip4_destination_matches_route (im, x, address, address_length))
1190           return clib_error_create ("failed to add %U which conflicts with %U for interface %U",
1191                                     format_ip4_address_and_length, address, address_length,
1192                                     format_ip4_address_and_length, x, ia->address_length,
1193                                     format_vnet_sw_if_index_name, vnm, sw_if_index);
1194       }));
1195     }
1196
1197   elts_before = pool_elts (lm->if_address_pool);
1198
1199   error = ip_interface_address_add_del
1200     (lm,
1201      sw_if_index,
1202      addr_fib,
1203      address_length,
1204      is_del,
1205      &if_address_index);
1206   if (error)
1207     goto done;
1208
1209   if (vnet_sw_interface_is_admin_up (vnm, sw_if_index) && insert_routes)
1210     {
1211       if (is_del)
1212         ip4_del_interface_routes (im, ip4_af.fib_index, address,
1213                                   address_length);
1214
1215       else
1216           ip4_add_interface_routes (sw_if_index,
1217                                     im, ip4_af.fib_index,
1218                                     pool_elt_at_index
1219                                     (lm->if_address_pool, if_address_index));
1220     }
1221
1222   /* If pool did not grow/shrink: add duplicate address. */
1223   if (elts_before != pool_elts (lm->if_address_pool))
1224     {
1225       ip4_add_del_interface_address_callback_t * cb;
1226       vec_foreach (cb, im->add_del_interface_address_callbacks)
1227         cb->function (im, cb->function_opaque, sw_if_index,
1228                       address, address_length,
1229                       if_address_index,
1230                       is_del);
1231     }
1232
1233  done:
1234   vec_free (addr_fib);
1235   return error;
1236 }
1237
1238 clib_error_t *
1239 ip4_add_del_interface_address (vlib_main_t * vm, u32 sw_if_index,
1240                                ip4_address_t * address, u32 address_length,
1241                                u32 is_del)
1242 {
1243   return ip4_add_del_interface_address_internal
1244     (vm, sw_if_index, address, address_length,
1245      /* redistribute */ 1,
1246      /* insert_routes */ 1,
1247      is_del);
1248 }
1249
1250 static clib_error_t *
1251 ip4_sw_interface_admin_up_down (vnet_main_t * vnm,
1252                                 u32 sw_if_index,
1253                                 u32 flags)
1254 {
1255   ip4_main_t * im = &ip4_main;
1256   ip_interface_address_t * ia;
1257   ip4_address_t * a;
1258   u32 is_admin_up, fib_index;
1259
1260   /* Fill in lookup tables with default table (0). */
1261   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1262
1263   vec_validate_init_empty (im->lookup_main.if_address_pool_index_by_sw_if_index, sw_if_index, ~0);
1264
1265   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1266
1267   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
1268
1269   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
1270                                 0 /* honor unnumbered */,
1271   ({
1272     a = ip_interface_address_get_address (&im->lookup_main, ia);
1273     if (is_admin_up)
1274       ip4_add_interface_routes (sw_if_index,
1275                                 im, fib_index,
1276                                 ia);
1277     else
1278       ip4_del_interface_routes (im, fib_index,
1279                                 a, ia->address_length);
1280   }));
1281
1282   return 0;
1283 }
1284
1285 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip4_sw_interface_admin_up_down);
1286
1287 /* Built-in ip4 unicast rx feature path definition */
1288 VNET_IP4_UNICAST_FEATURE_INIT (ip4_inacl, static) = {
1289   .node_name = "ip4-inacl",
1290   .runs_before = {"ip4-source-check-via-rx", 0},
1291   .feature_index = &ip4_main.ip4_unicast_rx_feature_check_access,
1292 };
1293
1294 VNET_IP4_UNICAST_FEATURE_INIT (ip4_source_check_1, static) = {
1295   .node_name = "ip4-source-check-via-rx",
1296   .runs_before = {"ip4-source-check-via-any", 0},
1297   .feature_index =
1298   &ip4_main.ip4_unicast_rx_feature_source_reachable_via_rx,
1299 };
1300
1301 VNET_IP4_UNICAST_FEATURE_INIT (ip4_source_check_2, static) = {
1302   .node_name = "ip4-source-check-via-any",
1303   .runs_before = {"ipsec-input-ip4", 0},
1304   .feature_index =
1305   &ip4_main.ip4_unicast_rx_feature_source_reachable_via_any,
1306 };
1307
1308 VNET_IP4_UNICAST_FEATURE_INIT (ip4_ipsec, static) = {
1309   .node_name = "ipsec-input-ip4",
1310   .runs_before = {"vpath-input-ip4", 0},
1311   .feature_index = &ip4_main.ip4_unicast_rx_feature_ipsec,
1312 };
1313
1314 VNET_IP4_UNICAST_FEATURE_INIT (ip4_vpath, static) = {
1315   .node_name = "vpath-input-ip4",
1316   .runs_before = {"ip4-lookup", 0},
1317   .feature_index = &ip4_main.ip4_unicast_rx_feature_vpath,
1318 };
1319
1320 VNET_IP4_UNICAST_FEATURE_INIT (ip4_lookup, static) = {
1321   .node_name = "ip4-lookup",
1322   .runs_before = {0}, /* not before any other features */
1323   .feature_index = &ip4_main.ip4_unicast_rx_feature_lookup,
1324 };
1325
1326 /* Built-in ip4 multicast rx feature path definition */
1327 VNET_IP4_MULTICAST_FEATURE_INIT (ip4_vpath_mc, static) = {
1328   .node_name = "vpath-input-ip4",
1329   .runs_before = {"ip4-lookup-multicast", 0},
1330   .feature_index = &ip4_main.ip4_multicast_rx_feature_vpath,
1331 };
1332
1333 VNET_IP4_MULTICAST_FEATURE_INIT (ip4_lookup_mc, static) = {
1334   .node_name = "ip4-lookup-multicast",
1335   .runs_before = {0}, /* not before any other features */
1336   .feature_index = &ip4_main.ip4_multicast_rx_feature_lookup,
1337 };
1338
1339 static char * feature_start_nodes[] =
1340   { "ip4-input", "ip4-input-no-checksum"};
1341
1342 static clib_error_t *
1343 ip4_feature_init (vlib_main_t * vm, ip4_main_t * im)
1344 {
1345   ip_lookup_main_t * lm = &im->lookup_main;
1346   clib_error_t * error;
1347   vnet_cast_t cast;
1348
1349   for (cast = 0; cast < VNET_N_CAST; cast++)
1350     {
1351       ip_config_main_t * cm = &lm->rx_config_mains[cast];
1352       vnet_config_main_t * vcm = &cm->config_main;
1353
1354       if ((error = ip_feature_init_cast (vm, cm, vcm,
1355                                          feature_start_nodes,
1356                                          ARRAY_LEN(feature_start_nodes),
1357                                          cast,
1358                                          1 /* is_ip4 */)))
1359         return error;
1360     }
1361   return 0;
1362 }
1363
1364 static clib_error_t *
1365 ip4_sw_interface_add_del (vnet_main_t * vnm,
1366                           u32 sw_if_index,
1367                           u32 is_add)
1368 {
1369   vlib_main_t * vm = vnm->vlib_main;
1370   ip4_main_t * im = &ip4_main;
1371   ip_lookup_main_t * lm = &im->lookup_main;
1372   u32 ci, cast;
1373   u32 feature_index;
1374
1375   for (cast = 0; cast < VNET_N_CAST; cast++)
1376     {
1377       ip_config_main_t * cm = &lm->rx_config_mains[cast];
1378       vnet_config_main_t * vcm = &cm->config_main;
1379
1380       vec_validate_init_empty (cm->config_index_by_sw_if_index, sw_if_index, ~0);
1381       ci = cm->config_index_by_sw_if_index[sw_if_index];
1382
1383       if (cast == VNET_UNICAST)
1384         feature_index = im->ip4_unicast_rx_feature_lookup;
1385       else
1386         feature_index = im->ip4_multicast_rx_feature_lookup;
1387
1388       if (is_add)
1389         ci = vnet_config_add_feature (vm, vcm,
1390                                       ci,
1391                                       feature_index,
1392                                       /* config data */ 0,
1393                                       /* # bytes of config data */ 0);
1394       else
1395         ci = vnet_config_del_feature (vm, vcm,
1396                                       ci,
1397                                       feature_index,
1398                                       /* config data */ 0,
1399                                       /* # bytes of config data */ 0);
1400
1401       cm->config_index_by_sw_if_index[sw_if_index] = ci;
1402     }
1403
1404   return /* no error */ 0;
1405 }
1406
1407 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip4_sw_interface_add_del);
1408
1409 static u8 * format_ip4_lookup_trace (u8 * s, va_list * args);
1410
1411 VLIB_REGISTER_NODE (ip4_lookup_node) = {
1412   .function = ip4_lookup,
1413   .name = "ip4-lookup",
1414   .vector_size = sizeof (u32),
1415
1416   .format_trace = format_ip4_lookup_trace,
1417
1418   .n_next_nodes = IP4_LOOKUP_N_NEXT,
1419   .next_nodes = IP4_LOOKUP_NEXT_NODES,
1420 };
1421
1422 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup)
1423
1424 static uword
1425 ip4_indirect (vlib_main_t * vm,
1426                vlib_node_runtime_t * node,
1427                vlib_frame_t * frame)
1428 {
1429   return ip4_lookup_inline (vm, node, frame,
1430                             /* lookup_for_responses_to_locally_received_packets */ 0,
1431                             /* is_indirect */ 1);
1432 }
1433
1434 VLIB_REGISTER_NODE (ip4_indirect_node) = {
1435   .function = ip4_indirect,
1436   .name = "ip4-indirect",
1437   .vector_size = sizeof (u32),
1438   .sibling_of = "ip4-lookup",
1439   .format_trace = format_ip4_lookup_trace,
1440
1441   .n_next_nodes = 0,
1442 };
1443
1444 VLIB_NODE_FUNCTION_MULTIARCH (ip4_indirect_node, ip4_indirect)
1445
1446
1447 /* Global IP4 main. */
1448 ip4_main_t ip4_main;
1449
1450 clib_error_t *
1451 ip4_lookup_init (vlib_main_t * vm)
1452 {
1453   ip4_main_t * im = &ip4_main;
1454   uword i;
1455
1456   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
1457     {
1458       u32 m;
1459
1460       if (i < 32)
1461         m = pow2_mask (i) << (32 - i);
1462       else
1463         m = ~0;
1464       im->fib_masks[i] = clib_host_to_net_u32 (m);
1465     }
1466
1467   /* Create FIB with index 0 and table id of 0. */
1468   find_ip4_fib_by_table_index_or_id (im, /* table id */ 0, IP4_ROUTE_FLAG_TABLE_ID);
1469
1470   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
1471
1472   {
1473     pg_node_t * pn;
1474     pn = pg_get_node (ip4_lookup_node.index);
1475     pn->unformat_edit = unformat_pg_ip4_header;
1476   }
1477
1478   {
1479     ethernet_arp_header_t h;
1480
1481     memset (&h, 0, sizeof (h));
1482
1483     /* Set target ethernet address to all zeros. */
1484     memset (h.ip4_over_ethernet[1].ethernet, 0, sizeof (h.ip4_over_ethernet[1].ethernet));
1485
1486 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
1487 #define _8(f,v) h.f = v;
1488     _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
1489     _16 (l3_type, ETHERNET_TYPE_IP4);
1490     _8 (n_l2_address_bytes, 6);
1491     _8 (n_l3_address_bytes, 4);
1492     _16 (opcode, ETHERNET_ARP_OPCODE_request);
1493 #undef _16
1494 #undef _8
1495
1496     vlib_packet_template_init (vm,
1497                                &im->ip4_arp_request_packet_template,
1498                                /* data */ &h,
1499                                sizeof (h),
1500                                /* alloc chunk size */ 8,
1501                                "ip4 arp");
1502   }
1503
1504   ip4_feature_init (vm, im);
1505
1506   return 0;
1507 }
1508
1509 VLIB_INIT_FUNCTION (ip4_lookup_init);
1510
1511 typedef struct {
1512   /* Adjacency taken. */
1513   u32 adj_index;
1514   u32 flow_hash;
1515   u32 fib_index;
1516
1517   /* Packet data, possibly *after* rewrite. */
1518   u8 packet_data[64 - 1*sizeof(u32)];
1519 } ip4_forward_next_trace_t;
1520
1521 static u8 * format_ip4_forward_next_trace (u8 * s, va_list * args)
1522 {
1523   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1524   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1525   ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1526   uword indent = format_get_indent (s);
1527   s = format (s, "%U%U",
1528                 format_white_space, indent,
1529                 format_ip4_header, t->packet_data);
1530   return s;
1531 }
1532
1533 static u8 * format_ip4_lookup_trace (u8 * s, va_list * args)
1534 {
1535   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1536   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1537   ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1538   vnet_main_t * vnm = vnet_get_main();
1539   ip4_main_t * im = &ip4_main;
1540   uword indent = format_get_indent (s);
1541
1542   s = format (s, "fib %d adj-idx %d : %U flow hash: 0x%08x",
1543               t->fib_index, t->adj_index, format_ip_adjacency,
1544               vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1545   s = format (s, "\n%U%U",
1546               format_white_space, indent,
1547               format_ip4_header, t->packet_data);
1548   return s;
1549 }
1550
1551 static u8 * format_ip4_rewrite_trace (u8 * s, va_list * args)
1552 {
1553   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1554   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1555   ip4_forward_next_trace_t * t = va_arg (*args, ip4_forward_next_trace_t *);
1556   vnet_main_t * vnm = vnet_get_main();
1557   ip4_main_t * im = &ip4_main;
1558   uword indent = format_get_indent (s);
1559
1560   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
1561               t->fib_index, t->adj_index, format_ip_adjacency,
1562               vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1563   s = format (s, "\n%U%U",
1564               format_white_space, indent,
1565               format_ip_adjacency_packet_data,
1566               vnm, &im->lookup_main, t->adj_index,
1567               t->packet_data, sizeof (t->packet_data));
1568   return s;
1569 }
1570
1571 /* Common trace function for all ip4-forward next nodes. */
1572 void
1573 ip4_forward_next_trace (vlib_main_t * vm,
1574                         vlib_node_runtime_t * node,
1575                         vlib_frame_t * frame,
1576                         vlib_rx_or_tx_t which_adj_index)
1577 {
1578   u32 * from, n_left;
1579   ip4_main_t * im = &ip4_main;
1580
1581   n_left = frame->n_vectors;
1582   from = vlib_frame_vector_args (frame);
1583
1584   while (n_left >= 4)
1585     {
1586       u32 bi0, bi1;
1587       vlib_buffer_t * b0, * b1;
1588       ip4_forward_next_trace_t * t0, * t1;
1589
1590       /* Prefetch next iteration. */
1591       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1592       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1593
1594       bi0 = from[0];
1595       bi1 = from[1];
1596
1597       b0 = vlib_get_buffer (vm, bi0);
1598       b1 = vlib_get_buffer (vm, bi1);
1599
1600       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1601         {
1602           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1603           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1604           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1605           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1606               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1607               vec_elt (im->fib_index_by_sw_if_index,
1608                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1609
1610           clib_memcpy (t0->packet_data,
1611                   vlib_buffer_get_current (b0),
1612                   sizeof (t0->packet_data));
1613         }
1614       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1615         {
1616           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1617           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1618           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1619           t1->fib_index = (vnet_buffer(b1)->sw_if_index[VLIB_TX] != (u32)~0) ?
1620               vnet_buffer(b1)->sw_if_index[VLIB_TX] :
1621               vec_elt (im->fib_index_by_sw_if_index,
1622                        vnet_buffer(b1)->sw_if_index[VLIB_RX]);
1623           clib_memcpy (t1->packet_data,
1624                   vlib_buffer_get_current (b1),
1625                   sizeof (t1->packet_data));
1626         }
1627       from += 2;
1628       n_left -= 2;
1629     }
1630
1631   while (n_left >= 1)
1632     {
1633       u32 bi0;
1634       vlib_buffer_t * b0;
1635       ip4_forward_next_trace_t * t0;
1636
1637       bi0 = from[0];
1638
1639       b0 = vlib_get_buffer (vm, bi0);
1640
1641       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1642         {
1643           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1644           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1645           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1646           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1647               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1648               vec_elt (im->fib_index_by_sw_if_index,
1649                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1650           clib_memcpy (t0->packet_data,
1651                   vlib_buffer_get_current (b0),
1652                   sizeof (t0->packet_data));
1653         }
1654       from += 1;
1655       n_left -= 1;
1656     }
1657 }
1658
1659 static uword
1660 ip4_drop_or_punt (vlib_main_t * vm,
1661                   vlib_node_runtime_t * node,
1662                   vlib_frame_t * frame,
1663                   ip4_error_t error_code)
1664 {
1665   u32 * buffers = vlib_frame_vector_args (frame);
1666   uword n_packets = frame->n_vectors;
1667
1668   vlib_error_drop_buffers (vm, node,
1669                            buffers,
1670                            /* stride */ 1,
1671                            n_packets,
1672                            /* next */ 0,
1673                            ip4_input_node.index,
1674                            error_code);
1675
1676   if (node->flags & VLIB_NODE_FLAG_TRACE)
1677     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1678
1679   return n_packets;
1680 }
1681
1682 static uword
1683 ip4_drop (vlib_main_t * vm,
1684           vlib_node_runtime_t * node,
1685           vlib_frame_t * frame)
1686 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_DROP); }
1687
1688 static uword
1689 ip4_punt (vlib_main_t * vm,
1690           vlib_node_runtime_t * node,
1691           vlib_frame_t * frame)
1692 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_PUNT); }
1693
1694 static uword
1695 ip4_miss (vlib_main_t * vm,
1696           vlib_node_runtime_t * node,
1697           vlib_frame_t * frame)
1698 { return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_DST_LOOKUP_MISS); }
1699
1700 VLIB_REGISTER_NODE (ip4_drop_node,static) = {
1701   .function = ip4_drop,
1702   .name = "ip4-drop",
1703   .vector_size = sizeof (u32),
1704
1705   .format_trace = format_ip4_forward_next_trace,
1706
1707   .n_next_nodes = 1,
1708   .next_nodes = {
1709     [0] = "error-drop",
1710   },
1711 };
1712
1713 VLIB_NODE_FUNCTION_MULTIARCH (ip4_drop_node, ip4_drop)
1714
1715 VLIB_REGISTER_NODE (ip4_punt_node,static) = {
1716   .function = ip4_punt,
1717   .name = "ip4-punt",
1718   .vector_size = sizeof (u32),
1719
1720   .format_trace = format_ip4_forward_next_trace,
1721
1722   .n_next_nodes = 1,
1723   .next_nodes = {
1724     [0] = "error-punt",
1725   },
1726 };
1727
1728 VLIB_NODE_FUNCTION_MULTIARCH (ip4_punt_node, ip4_punt)
1729
1730 VLIB_REGISTER_NODE (ip4_miss_node,static) = {
1731   .function = ip4_miss,
1732   .name = "ip4-miss",
1733   .vector_size = sizeof (u32),
1734
1735   .format_trace = format_ip4_forward_next_trace,
1736
1737   .n_next_nodes = 1,
1738   .next_nodes = {
1739     [0] = "error-drop",
1740   },
1741 };
1742
1743 VLIB_NODE_FUNCTION_MULTIARCH (ip4_miss_node, ip4_miss)
1744
1745 /* Compute TCP/UDP/ICMP4 checksum in software. */
1746 u16
1747 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1748                               ip4_header_t * ip0)
1749 {
1750   ip_csum_t sum0;
1751   u32 ip_header_length, payload_length_host_byte_order;
1752   u32 n_this_buffer, n_bytes_left;
1753   u16 sum16;
1754   void * data_this_buffer;
1755
1756   /* Initialize checksum with ip header. */
1757   ip_header_length = ip4_header_bytes (ip0);
1758   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->length) - ip_header_length;
1759   sum0 = clib_host_to_net_u32 (payload_length_host_byte_order + (ip0->protocol << 16));
1760
1761   if (BITS (uword) == 32)
1762     {
1763       sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u32));
1764       sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->dst_address, u32));
1765     }
1766   else
1767     sum0 = ip_csum_with_carry (sum0, clib_mem_unaligned (&ip0->src_address, u64));
1768
1769   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1770   data_this_buffer = (void *) ip0 + ip_header_length;
1771   if (n_this_buffer + ip_header_length > p0->current_length)
1772     n_this_buffer = p0->current_length > ip_header_length ? p0->current_length - ip_header_length : 0;
1773   while (1)
1774     {
1775       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1776       n_bytes_left -= n_this_buffer;
1777       if (n_bytes_left == 0)
1778         break;
1779
1780       ASSERT (p0->flags & VLIB_BUFFER_NEXT_PRESENT);
1781       p0 = vlib_get_buffer (vm, p0->next_buffer);
1782       data_this_buffer = vlib_buffer_get_current (p0);
1783       n_this_buffer = p0->current_length;
1784     }
1785
1786   sum16 = ~ ip_csum_fold (sum0);
1787
1788   return sum16;
1789 }
1790
1791 static u32
1792 ip4_tcp_udp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1793 {
1794   ip4_header_t * ip0 = vlib_buffer_get_current (p0);
1795   udp_header_t * udp0;
1796   u16 sum16;
1797
1798   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1799           || ip0->protocol == IP_PROTOCOL_UDP);
1800
1801   udp0 = (void *) (ip0 + 1);
1802   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1803     {
1804       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1805                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1806       return p0->flags;
1807     }
1808
1809   sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1810
1811   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1812                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1813
1814   return p0->flags;
1815 }
1816
1817 static uword
1818 ip4_local (vlib_main_t * vm,
1819            vlib_node_runtime_t * node,
1820            vlib_frame_t * frame)
1821 {
1822   ip4_main_t * im = &ip4_main;
1823   ip_lookup_main_t * lm = &im->lookup_main;
1824   ip_local_next_t next_index;
1825   u32 * from, * to_next, n_left_from, n_left_to_next;
1826   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip4_input_node.index);
1827
1828   from = vlib_frame_vector_args (frame);
1829   n_left_from = frame->n_vectors;
1830   next_index = node->cached_next_index;
1831
1832   if (node->flags & VLIB_NODE_FLAG_TRACE)
1833     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1834
1835   while (n_left_from > 0)
1836     {
1837       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1838
1839       while (n_left_from >= 4 && n_left_to_next >= 2)
1840         {
1841           vlib_buffer_t * p0, * p1;
1842           ip4_header_t * ip0, * ip1;
1843           udp_header_t * udp0, * udp1;
1844           ip4_fib_mtrie_t * mtrie0, * mtrie1;
1845           ip4_fib_mtrie_leaf_t leaf0, leaf1;
1846           ip_adjacency_t * adj0, * adj1;
1847           u32 pi0, ip_len0, udp_len0, flags0, next0, fib_index0, adj_index0;
1848           u32 pi1, ip_len1, udp_len1, flags1, next1, fib_index1, adj_index1;
1849           i32 len_diff0, len_diff1;
1850           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1851           u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1852           u8 enqueue_code;
1853
1854           pi0 = to_next[0] = from[0];
1855           pi1 = to_next[1] = from[1];
1856           from += 2;
1857           n_left_from -= 2;
1858           to_next += 2;
1859           n_left_to_next -= 2;
1860
1861           p0 = vlib_get_buffer (vm, pi0);
1862           p1 = vlib_get_buffer (vm, pi1);
1863
1864           ip0 = vlib_buffer_get_current (p0);
1865           ip1 = vlib_buffer_get_current (p1);
1866
1867           fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
1868                                 vnet_buffer(p0)->sw_if_index[VLIB_RX]);
1869           fib_index1 = vec_elt (im->fib_index_by_sw_if_index,
1870                                 vnet_buffer(p1)->sw_if_index[VLIB_RX]);
1871
1872           mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
1873           mtrie1 = &vec_elt_at_index (im->fibs, fib_index1)->mtrie;
1874
1875           leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
1876
1877           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 0);
1878           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 0);
1879
1880           /* Treat IP frag packets as "experimental" protocol for now
1881              until support of IP frag reassembly is implemented */
1882           proto0 = ip4_is_fragment(ip0) ? 0xfe : ip0->protocol;
1883           proto1 = ip4_is_fragment(ip1) ? 0xfe : ip1->protocol;
1884           is_udp0 = proto0 == IP_PROTOCOL_UDP;
1885           is_udp1 = proto1 == IP_PROTOCOL_UDP;
1886           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1887           is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1888
1889           flags0 = p0->flags;
1890           flags1 = p1->flags;
1891
1892           good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1893           good_tcp_udp1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1894
1895           udp0 = ip4_next_header (ip0);
1896           udp1 = ip4_next_header (ip1);
1897
1898           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1899           good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1900           good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1901
1902           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
1903           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 1);
1904
1905           /* Verify UDP length. */
1906           ip_len0 = clib_net_to_host_u16 (ip0->length);
1907           ip_len1 = clib_net_to_host_u16 (ip1->length);
1908           udp_len0 = clib_net_to_host_u16 (udp0->length);
1909           udp_len1 = clib_net_to_host_u16 (udp1->length);
1910
1911           len_diff0 = ip_len0 - udp_len0;
1912           len_diff1 = ip_len1 - udp_len1;
1913
1914           len_diff0 = is_udp0 ? len_diff0 : 0;
1915           len_diff1 = is_udp1 ? len_diff1 : 0;
1916
1917           if (PREDICT_FALSE (! (is_tcp_udp0 & is_tcp_udp1
1918                                 & good_tcp_udp0 & good_tcp_udp1)))
1919             {
1920               if (is_tcp_udp0)
1921                 {
1922                   if (is_tcp_udp0
1923                       && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1924                     flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
1925                   good_tcp_udp0 =
1926                     (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1927                   good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1928                 }
1929               if (is_tcp_udp1)
1930                 {
1931                   if (is_tcp_udp1
1932                       && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1933                     flags1 = ip4_tcp_udp_validate_checksum (vm, p1);
1934                   good_tcp_udp1 =
1935                     (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1936                   good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1937                 }
1938             }
1939
1940           good_tcp_udp0 &= len_diff0 >= 0;
1941           good_tcp_udp1 &= len_diff1 >= 0;
1942
1943           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1944           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 2);
1945
1946           error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1947
1948           error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
1949           error1 = len_diff1 < 0 ? IP4_ERROR_UDP_LENGTH : error1;
1950
1951           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1952           error0 = (is_tcp_udp0 && ! good_tcp_udp0
1953                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0
1954                     : error0);
1955           error1 = (is_tcp_udp1 && ! good_tcp_udp1
1956                     ? IP4_ERROR_TCP_CHECKSUM + is_udp1
1957                     : error1);
1958
1959           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1960           leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 3);
1961
1962           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1963           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
1964
1965           vnet_buffer (p1)->ip.adj_index[VLIB_RX] = adj_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
1966           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
1967
1968           ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
1969                                                            &ip0->src_address,
1970                                                            /* no_default_route */ 1));
1971           ASSERT (adj_index1 == ip4_fib_lookup_with_table (im, fib_index1,
1972                                                            &ip1->src_address,
1973                                                            /* no_default_route */ 1));
1974
1975           adj0 = ip_get_adjacency (lm, adj_index0);
1976           adj1 = ip_get_adjacency (lm, adj_index1);
1977
1978           /*
1979            * Must have a route to source otherwise we drop the packet.
1980            * ip4 broadcasts are accepted, e.g. to make dhcp client work
1981            */
1982           error0 = (error0 == IP4_ERROR_UNKNOWN_PROTOCOL
1983                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_REWRITE
1984                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP
1985                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_LOCAL
1986                     && ip0->dst_address.as_u32 != 0xFFFFFFFF
1987                     ? IP4_ERROR_SRC_LOOKUP_MISS
1988                     : error0);
1989           error1 = (error1 == IP4_ERROR_UNKNOWN_PROTOCOL
1990                     && adj1->lookup_next_index != IP_LOOKUP_NEXT_REWRITE
1991                     && adj1->lookup_next_index != IP_LOOKUP_NEXT_ARP
1992                     && adj1->lookup_next_index != IP_LOOKUP_NEXT_LOCAL
1993                     && ip0->dst_address.as_u32 != 0xFFFFFFFF
1994                     ? IP4_ERROR_SRC_LOOKUP_MISS
1995                     : error1);
1996
1997           next0 = lm->local_next_by_ip_protocol[proto0];
1998           next1 = lm->local_next_by_ip_protocol[proto1];
1999
2000           next0 = error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
2001           next1 = error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
2002
2003           p0->error = error0 ? error_node->errors[error0] : 0;
2004           p1->error = error1 ? error_node->errors[error1] : 0;
2005
2006           enqueue_code = (next0 != next_index) + 2*(next1 != next_index);
2007
2008           if (PREDICT_FALSE (enqueue_code != 0))
2009             {
2010               switch (enqueue_code)
2011                 {
2012                 case 1:
2013                   /* A B A */
2014                   to_next[-2] = pi1;
2015                   to_next -= 1;
2016                   n_left_to_next += 1;
2017                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
2018                   break;
2019
2020                 case 2:
2021                   /* A A B */
2022                   to_next -= 1;
2023                   n_left_to_next += 1;
2024                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
2025                   break;
2026
2027                 case 3:
2028                   /* A B B or A B C */
2029                   to_next -= 2;
2030                   n_left_to_next += 2;
2031                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
2032                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
2033                   if (next0 == next1)
2034                     {
2035                       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2036                       next_index = next1;
2037                       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2038                     }
2039                   break;
2040                 }
2041             }
2042         }
2043
2044       while (n_left_from > 0 && n_left_to_next > 0)
2045         {
2046           vlib_buffer_t * p0;
2047           ip4_header_t * ip0;
2048           udp_header_t * udp0;
2049           ip4_fib_mtrie_t * mtrie0;
2050           ip4_fib_mtrie_leaf_t leaf0;
2051           ip_adjacency_t * adj0;
2052           u32 pi0, next0, ip_len0, udp_len0, flags0, fib_index0, adj_index0;
2053           i32 len_diff0;
2054           u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
2055
2056           pi0 = to_next[0] = from[0];
2057           from += 1;
2058           n_left_from -= 1;
2059           to_next += 1;
2060           n_left_to_next -= 1;
2061
2062           p0 = vlib_get_buffer (vm, pi0);
2063
2064           ip0 = vlib_buffer_get_current (p0);
2065
2066           fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
2067                                 vnet_buffer(p0)->sw_if_index[VLIB_RX]);
2068
2069           mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
2070
2071           leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
2072
2073           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 0);
2074
2075           /* Treat IP frag packets as "experimental" protocol for now
2076              until support of IP frag reassembly is implemented */
2077           proto0 = ip4_is_fragment(ip0) ? 0xfe : ip0->protocol;
2078           is_udp0 = proto0 == IP_PROTOCOL_UDP;
2079           is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
2080
2081           flags0 = p0->flags;
2082
2083           good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
2084
2085           udp0 = ip4_next_header (ip0);
2086
2087           /* Don't verify UDP checksum for packets with explicit zero checksum. */
2088           good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
2089
2090           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
2091
2092           /* Verify UDP length. */
2093           ip_len0 = clib_net_to_host_u16 (ip0->length);
2094           udp_len0 = clib_net_to_host_u16 (udp0->length);
2095
2096           len_diff0 = ip_len0 - udp_len0;
2097
2098           len_diff0 = is_udp0 ? len_diff0 : 0;
2099
2100           if (PREDICT_FALSE (! (is_tcp_udp0 & good_tcp_udp0)))
2101             {
2102               if (is_tcp_udp0)
2103                 {
2104                   if (is_tcp_udp0
2105                       && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
2106                     flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
2107                   good_tcp_udp0 =
2108                     (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
2109                   good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
2110                 }
2111             }
2112
2113           good_tcp_udp0 &= len_diff0 >= 0;
2114
2115           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
2116
2117           error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
2118
2119           error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
2120
2121           ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
2122           error0 = (is_tcp_udp0 && ! good_tcp_udp0
2123                     ? IP4_ERROR_TCP_CHECKSUM + is_udp0
2124                     : error0);
2125
2126           leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
2127
2128           vnet_buffer (p0)->ip.adj_index[VLIB_RX] = adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2129           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
2130
2131           ASSERT (adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
2132                                                            &ip0->src_address,
2133                                                            /* no_default_route */ 1));
2134
2135           adj0 = ip_get_adjacency (lm, adj_index0);
2136
2137           /* Must have a route to source otherwise we drop the packet. */
2138           error0 = (error0 == IP4_ERROR_UNKNOWN_PROTOCOL
2139                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_REWRITE
2140                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP
2141                     && adj0->lookup_next_index != IP_LOOKUP_NEXT_LOCAL
2142                     && ip0->dst_address.as_u32 != 0xFFFFFFFF
2143                     ? IP4_ERROR_SRC_LOOKUP_MISS
2144                     : error0);
2145
2146           next0 = lm->local_next_by_ip_protocol[proto0];
2147
2148           next0 = error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
2149
2150           p0->error = error0? error_node->errors[error0] : 0;
2151
2152           if (PREDICT_FALSE (next0 != next_index))
2153             {
2154               n_left_to_next += 1;
2155               vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2156
2157               next_index = next0;
2158               vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2159               to_next[0] = pi0;
2160               to_next += 1;
2161               n_left_to_next -= 1;
2162             }
2163         }
2164
2165       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2166     }
2167
2168   return frame->n_vectors;
2169 }
2170
2171 VLIB_REGISTER_NODE (ip4_local_node,static) = {
2172   .function = ip4_local,
2173   .name = "ip4-local",
2174   .vector_size = sizeof (u32),
2175
2176   .format_trace = format_ip4_forward_next_trace,
2177
2178   .n_next_nodes = IP_LOCAL_N_NEXT,
2179   .next_nodes = {
2180     [IP_LOCAL_NEXT_DROP] = "error-drop",
2181     [IP_LOCAL_NEXT_PUNT] = "error-punt",
2182     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
2183     [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",
2184   },
2185 };
2186
2187 VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local)
2188
2189 void ip4_register_protocol (u32 protocol, u32 node_index)
2190 {
2191   vlib_main_t * vm = vlib_get_main();
2192   ip4_main_t * im = &ip4_main;
2193   ip_lookup_main_t * lm = &im->lookup_main;
2194
2195   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
2196   lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip4_local_node.index, node_index);
2197 }
2198
2199 static clib_error_t *
2200 show_ip_local_command_fn (vlib_main_t * vm,
2201                           unformat_input_t * input,
2202                          vlib_cli_command_t * cmd)
2203 {
2204   ip4_main_t * im = &ip4_main;
2205   ip_lookup_main_t * lm = &im->lookup_main;
2206   int i;
2207
2208   vlib_cli_output (vm, "Protocols handled by ip4_local");
2209   for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
2210     {
2211       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
2212         vlib_cli_output (vm, "%d", i);
2213     }
2214   return 0;
2215 }
2216
2217
2218
2219 VLIB_CLI_COMMAND (show_ip_local, static) = {
2220   .path = "show ip local",
2221   .function = show_ip_local_command_fn,
2222   .short_help = "Show ip local protocol table",
2223 };
2224
2225 static uword
2226 ip4_arp (vlib_main_t * vm,
2227          vlib_node_runtime_t * node,
2228          vlib_frame_t * frame)
2229 {
2230   vnet_main_t * vnm = vnet_get_main();
2231   ip4_main_t * im = &ip4_main;
2232   ip_lookup_main_t * lm = &im->lookup_main;
2233   u32 * from, * to_next_drop;
2234   uword n_left_from, n_left_to_next_drop, next_index;
2235   static f64 time_last_seed_change = -1e100;
2236   static u32 hash_seeds[3];
2237   static uword hash_bitmap[256 / BITS (uword)];
2238   f64 time_now;
2239
2240   if (node->flags & VLIB_NODE_FLAG_TRACE)
2241     ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2242
2243   time_now = vlib_time_now (vm);
2244   if (time_now - time_last_seed_change > 1e-3)
2245     {
2246       uword i;
2247       u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
2248                                              sizeof (hash_seeds));
2249       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
2250         hash_seeds[i] = r[i];
2251
2252       /* Mark all hash keys as been no-seen before. */
2253       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
2254         hash_bitmap[i] = 0;
2255
2256       time_last_seed_change = time_now;
2257     }
2258
2259   from = vlib_frame_vector_args (frame);
2260   n_left_from = frame->n_vectors;
2261   next_index = node->cached_next_index;
2262   if (next_index == IP4_ARP_NEXT_DROP)
2263     next_index = IP4_ARP_N_NEXT; /* point to first interface */
2264
2265   while (n_left_from > 0)
2266     {
2267       vlib_get_next_frame (vm, node, IP4_ARP_NEXT_DROP,
2268                            to_next_drop, n_left_to_next_drop);
2269
2270       while (n_left_from > 0 && n_left_to_next_drop > 0)
2271         {
2272           vlib_buffer_t * p0;
2273           ip4_header_t * ip0;
2274           ethernet_header_t * eh0;
2275           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
2276           uword bm0;
2277           ip_adjacency_t * adj0;
2278
2279           pi0 = from[0];
2280
2281           p0 = vlib_get_buffer (vm, pi0);
2282
2283           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2284           adj0 = ip_get_adjacency (lm, adj_index0);
2285           ip0 = vlib_buffer_get_current (p0);
2286
2287           /* If packet destination is not local, send ARP to next hop */
2288           if (adj0->arp.next_hop.ip4.as_u32)
2289             ip0->dst_address.data_u32 = adj0->arp.next_hop.ip4.as_u32;
2290
2291           /*
2292            * if ip4_rewrite_local applied the IP_LOOKUP_NEXT_ARP
2293            * rewrite to this packet, we need to skip it here.
2294            * Note, to distinguish from src IP addr *.8.6.*, we
2295            * check for a bcast eth dest instead of IPv4 version.
2296            */
2297           eh0 = (ethernet_header_t*)ip0;
2298           if ((ip0->ip_version_and_header_length & 0xF0) != 0x40)
2299             {
2300               u32 vlan_num = 0;
2301               u16 * etype = &eh0->type;
2302               while ((*etype == clib_host_to_net_u16 (0x8100)) //dot1q
2303                   || (*etype == clib_host_to_net_u16 (0x88a8)))//dot1ad
2304                 {
2305                   vlan_num += 1;
2306                   etype += 2; //vlan tag also 16 bits, same as etype
2307                 }
2308               if (*etype == clib_host_to_net_u16 (0x0806))     //arp
2309                 {
2310                   vlib_buffer_advance (
2311                       p0, sizeof(ethernet_header_t) + (4*vlan_num));
2312                   ip0 = vlib_buffer_get_current (p0);
2313                 }
2314             }
2315
2316           a0 = hash_seeds[0];
2317           b0 = hash_seeds[1];
2318           c0 = hash_seeds[2];
2319
2320           sw_if_index0 = adj0->rewrite_header.sw_if_index;
2321           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
2322
2323           a0 ^= ip0->dst_address.data_u32;
2324           b0 ^= sw_if_index0;
2325
2326           hash_v3_finalize32 (a0, b0, c0);
2327
2328           c0 &= BITS (hash_bitmap) - 1;
2329           c0 = c0 / BITS (uword);
2330           m0 = (uword) 1 << (c0 % BITS (uword));
2331
2332           bm0 = hash_bitmap[c0];
2333           drop0 = (bm0 & m0) != 0;
2334
2335           /* Mark it as seen. */
2336           hash_bitmap[c0] = bm0 | m0;
2337
2338           from += 1;
2339           n_left_from -= 1;
2340           to_next_drop[0] = pi0;
2341           to_next_drop += 1;
2342           n_left_to_next_drop -= 1;
2343
2344           p0->error = node->errors[drop0 ? IP4_ARP_ERROR_DROP : IP4_ARP_ERROR_REQUEST_SENT];
2345
2346           if (drop0)
2347             continue;
2348
2349           /*
2350            * Can happen if the control-plane is programming tables
2351            * with traffic flowing; at least that's today's lame excuse.
2352            */
2353           if (adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP)
2354             {
2355               p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
2356             }
2357           else
2358           /* Send ARP request. */
2359           {
2360             u32 bi0 = 0;
2361             vlib_buffer_t * b0;
2362             ethernet_arp_header_t * h0;
2363             vnet_hw_interface_t * hw_if0;
2364
2365             h0 = vlib_packet_template_get_packet (vm, &im->ip4_arp_request_packet_template, &bi0);
2366
2367             /* Add rewrite/encap string for ARP packet. */
2368             vnet_rewrite_one_header (adj0[0], h0, sizeof (ethernet_header_t));
2369
2370             hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2371
2372             /* Src ethernet address in ARP header. */
2373             clib_memcpy (h0->ip4_over_ethernet[0].ethernet, hw_if0->hw_address,
2374                     sizeof (h0->ip4_over_ethernet[0].ethernet));
2375
2376             ip4_src_address_for_packet (im, p0, &h0->ip4_over_ethernet[0].ip4, sw_if_index0);
2377
2378             /* Copy in destination address we are requesting. */
2379             h0->ip4_over_ethernet[1].ip4.data_u32 = ip0->dst_address.data_u32;
2380
2381             vlib_buffer_copy_trace_flag (vm, p0, bi0);
2382             b0 = vlib_get_buffer (vm, bi0);
2383             vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
2384
2385             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2386
2387             vlib_set_next_frame_buffer (vm, node, adj0->rewrite_header.next_index, bi0);
2388           }
2389         }
2390
2391       vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
2392     }
2393
2394   return frame->n_vectors;
2395 }
2396
2397 static char * ip4_arp_error_strings[] = {
2398   [IP4_ARP_ERROR_DROP] = "address overflow drops",
2399   [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
2400   [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
2401   [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
2402   [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
2403 };
2404
2405 VLIB_REGISTER_NODE (ip4_arp_node) = {
2406   .function = ip4_arp,
2407   .name = "ip4-arp",
2408   .vector_size = sizeof (u32),
2409
2410   .format_trace = format_ip4_forward_next_trace,
2411
2412   .n_errors = ARRAY_LEN (ip4_arp_error_strings),
2413   .error_strings = ip4_arp_error_strings,
2414
2415   .n_next_nodes = IP4_ARP_N_NEXT,
2416   .next_nodes = {
2417     [IP4_ARP_NEXT_DROP] = "error-drop",
2418   },
2419 };
2420
2421 #define foreach_notrace_ip4_arp_error           \
2422 _(DROP)                                         \
2423 _(REQUEST_SENT)                                 \
2424 _(REPLICATE_DROP)                               \
2425 _(REPLICATE_FAIL)
2426
2427 clib_error_t * arp_notrace_init (vlib_main_t * vm)
2428 {
2429   vlib_node_runtime_t *rt =
2430     vlib_node_get_runtime (vm, ip4_arp_node.index);
2431
2432   /* don't trace ARP request packets */
2433 #define _(a)                                    \
2434     vnet_pcap_drop_trace_filter_add_del         \
2435         (rt->errors[IP4_ARP_ERROR_##a],         \
2436          1 /* is_add */);
2437     foreach_notrace_ip4_arp_error;
2438 #undef _
2439   return 0;
2440 }
2441
2442 VLIB_INIT_FUNCTION(arp_notrace_init);
2443
2444
2445 /* Send an ARP request to see if given destination is reachable on given interface. */
2446 clib_error_t *
2447 ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
2448 {
2449   vnet_main_t * vnm = vnet_get_main();
2450   ip4_main_t * im = &ip4_main;
2451   ethernet_arp_header_t * h;
2452   ip4_address_t * src;
2453   ip_interface_address_t * ia;
2454   ip_adjacency_t * adj;
2455   vnet_hw_interface_t * hi;
2456   vnet_sw_interface_t * si;
2457   vlib_buffer_t * b;
2458   u32 bi = 0;
2459
2460   si = vnet_get_sw_interface (vnm, sw_if_index);
2461
2462   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2463     {
2464       return clib_error_return (0, "%U: interface %U down",
2465                                 format_ip4_address, dst,
2466                                 format_vnet_sw_if_index_name, vnm,
2467                                 sw_if_index);
2468     }
2469
2470   src = ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2471   if (! src)
2472     {
2473       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2474       return clib_error_return
2475         (0, "no matching interface address for destination %U (interface %U)",
2476          format_ip4_address, dst,
2477          format_vnet_sw_if_index_name, vnm, sw_if_index);
2478     }
2479
2480   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
2481
2482   h = vlib_packet_template_get_packet (vm, &im->ip4_arp_request_packet_template, &bi);
2483
2484   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2485
2486   clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address, sizeof (h->ip4_over_ethernet[0].ethernet));
2487
2488   h->ip4_over_ethernet[0].ip4 = src[0];
2489   h->ip4_over_ethernet[1].ip4 = dst[0];
2490
2491   b = vlib_get_buffer (vm, bi);
2492   vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2493
2494   /* Add encapsulation string for software interface (e.g. ethernet header). */
2495   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2496   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2497
2498   {
2499     vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
2500     u32 * to_next = vlib_frame_vector_args (f);
2501     to_next[0] = bi;
2502     f->n_vectors = 1;
2503     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2504   }
2505
2506   return /* no error */ 0;
2507 }
2508
2509 typedef enum {
2510   IP4_REWRITE_NEXT_DROP,
2511   IP4_REWRITE_NEXT_ARP,
2512 } ip4_rewrite_next_t;
2513
2514 always_inline uword
2515 ip4_rewrite_inline (vlib_main_t * vm,
2516                     vlib_node_runtime_t * node,
2517                     vlib_frame_t * frame,
2518                     int rewrite_for_locally_received_packets)
2519 {
2520   ip_lookup_main_t * lm = &ip4_main.lookup_main;
2521   u32 * from = vlib_frame_vector_args (frame);
2522   u32 n_left_from, n_left_to_next, * to_next, next_index;
2523   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip4_input_node.index);
2524   vlib_rx_or_tx_t adj_rx_tx = rewrite_for_locally_received_packets ? VLIB_RX : VLIB_TX;
2525
2526   n_left_from = frame->n_vectors;
2527   next_index = node->cached_next_index;
2528   u32 cpu_index = os_get_cpu_number();
2529
2530   while (n_left_from > 0)
2531     {
2532       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2533
2534       while (n_left_from >= 4 && n_left_to_next >= 2)
2535         {
2536           ip_adjacency_t * adj0, * adj1;
2537           vlib_buffer_t * p0, * p1;
2538           ip4_header_t * ip0, * ip1;
2539           u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
2540           u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
2541           u32 next0_override, next1_override;
2542
2543           if (rewrite_for_locally_received_packets)
2544               next0_override = next1_override = 0;
2545
2546           /* Prefetch next iteration. */
2547           {
2548             vlib_buffer_t * p2, * p3;
2549
2550             p2 = vlib_get_buffer (vm, from[2]);
2551             p3 = vlib_get_buffer (vm, from[3]);
2552
2553             vlib_prefetch_buffer_header (p2, STORE);
2554             vlib_prefetch_buffer_header (p3, STORE);
2555
2556             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2557             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2558           }
2559
2560           pi0 = to_next[0] = from[0];
2561           pi1 = to_next[1] = from[1];
2562
2563           from += 2;
2564           n_left_from -= 2;
2565           to_next += 2;
2566           n_left_to_next -= 2;
2567
2568           p0 = vlib_get_buffer (vm, pi0);
2569           p1 = vlib_get_buffer (vm, pi1);
2570
2571           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2572           adj_index1 = vnet_buffer (p1)->ip.adj_index[adj_rx_tx];
2573
2574           /* We should never rewrite a pkt using the MISS adjacency */
2575           ASSERT(adj_index0 && adj_index1);
2576
2577           ip0 = vlib_buffer_get_current (p0);
2578           ip1 = vlib_buffer_get_current (p1);
2579
2580           error0 = error1 = IP4_ERROR_NONE;
2581
2582           /* Decrement TTL & update checksum.
2583              Works either endian, so no need for byte swap. */
2584           if (! rewrite_for_locally_received_packets)
2585             {
2586               i32 ttl0 = ip0->ttl, ttl1 = ip1->ttl;
2587
2588               /* Input node should have reject packets with ttl 0. */
2589               ASSERT (ip0->ttl > 0);
2590               ASSERT (ip1->ttl > 0);
2591
2592               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2593               checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2594
2595               checksum0 += checksum0 >= 0xffff;
2596               checksum1 += checksum1 >= 0xffff;
2597
2598               ip0->checksum = checksum0;
2599               ip1->checksum = checksum1;
2600
2601               ttl0 -= 1;
2602               ttl1 -= 1;
2603
2604               ip0->ttl = ttl0;
2605               ip1->ttl = ttl1;
2606
2607               error0 = ttl0 <= 0 ? IP4_ERROR_TIME_EXPIRED : error0;
2608               error1 = ttl1 <= 0 ? IP4_ERROR_TIME_EXPIRED : error1;
2609
2610               /* Verify checksum. */
2611               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2612               ASSERT (ip1->checksum == ip4_header_checksum (ip1));
2613             }
2614
2615           /* Rewrite packet header and updates lengths. */
2616           adj0 = ip_get_adjacency (lm, adj_index0);
2617           adj1 = ip_get_adjacency (lm, adj_index1);
2618
2619           if (rewrite_for_locally_received_packets)
2620             {
2621               /*
2622                * If someone sends e.g. an icmp4 w/ src = dst = interface addr,
2623                * we end up here with a local adjacency in hand
2624                * The local adj rewrite data is 0xfefe on purpose.
2625                * Bad engineer, no donut for you.
2626                */
2627               if (PREDICT_FALSE(adj0->lookup_next_index
2628                                 == IP_LOOKUP_NEXT_LOCAL))
2629                 error0 = IP4_ERROR_SPOOFED_LOCAL_PACKETS;
2630               if (PREDICT_FALSE(adj0->lookup_next_index
2631                                 == IP_LOOKUP_NEXT_ARP))
2632                 next0_override = IP4_REWRITE_NEXT_ARP;
2633               if (PREDICT_FALSE(adj1->lookup_next_index
2634                                 == IP_LOOKUP_NEXT_LOCAL))
2635                 error1 = IP4_ERROR_SPOOFED_LOCAL_PACKETS;
2636               if (PREDICT_FALSE(adj1->lookup_next_index
2637                                 == IP_LOOKUP_NEXT_ARP))
2638                 next1_override = IP4_REWRITE_NEXT_ARP;
2639             }
2640
2641           /* Worth pipelining. No guarantee that adj0,1 are hot... */
2642           rw_len0 = adj0[0].rewrite_header.data_bytes;
2643           rw_len1 = adj1[0].rewrite_header.data_bytes;
2644
2645           /* Check MTU of outgoing interface. */
2646           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2647                     ? IP4_ERROR_MTU_EXCEEDED
2648                     : error0);
2649           error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
2650                     ? IP4_ERROR_MTU_EXCEEDED
2651                     : error1);
2652
2653           next0 = (error0 == IP4_ERROR_NONE)
2654             ? adj0[0].rewrite_header.next_index : 0;
2655
2656           if (rewrite_for_locally_received_packets)
2657               next0 = next0 && next0_override ? next0_override : next0;
2658
2659           next1 = (error1 == IP4_ERROR_NONE)
2660             ? adj1[0].rewrite_header.next_index : 0;
2661
2662           if (rewrite_for_locally_received_packets)
2663               next1 = next1 && next1_override ? next1_override : next1;
2664
2665           /*
2666            * We've already accounted for an ethernet_header_t elsewhere
2667            */
2668           if (PREDICT_FALSE (rw_len0 > sizeof(ethernet_header_t)))
2669               vlib_increment_combined_counter
2670                   (&lm->adjacency_counters,
2671                    cpu_index, adj_index0,
2672                    /* packet increment */ 0,
2673                    /* byte increment */ rw_len0-sizeof(ethernet_header_t));
2674
2675           if (PREDICT_FALSE (rw_len1 > sizeof(ethernet_header_t)))
2676               vlib_increment_combined_counter
2677                   (&lm->adjacency_counters,
2678                    cpu_index, adj_index1,
2679                    /* packet increment */ 0,
2680                    /* byte increment */ rw_len1-sizeof(ethernet_header_t));
2681
2682           p0->current_data -= rw_len0;
2683           p1->current_data -= rw_len1;
2684
2685           p0->current_length += rw_len0;
2686           p1->current_length += rw_len1;
2687
2688           vnet_buffer (p0)->sw_if_index[VLIB_TX] = adj0[0].rewrite_header.sw_if_index;
2689           vnet_buffer (p1)->sw_if_index[VLIB_TX] = adj1[0].rewrite_header.sw_if_index;
2690
2691           p0->error = error_node->errors[error0];
2692           p1->error = error_node->errors[error1];
2693
2694           /* Guess we are only writing on simple Ethernet header. */
2695           vnet_rewrite_two_headers (adj0[0], adj1[0],
2696                                     ip0, ip1,
2697                                     sizeof (ethernet_header_t));
2698
2699           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2700                                            to_next, n_left_to_next,
2701                                            pi0, pi1, next0, next1);
2702         }
2703
2704       while (n_left_from > 0 && n_left_to_next > 0)
2705         {
2706           ip_adjacency_t * adj0;
2707           vlib_buffer_t * p0;
2708           ip4_header_t * ip0;
2709           u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2710           u32 next0_override;
2711
2712           if (rewrite_for_locally_received_packets)
2713               next0_override = 0;
2714
2715           pi0 = to_next[0] = from[0];
2716
2717           p0 = vlib_get_buffer (vm, pi0);
2718
2719           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2720
2721           /* We should never rewrite a pkt using the MISS adjacency */
2722           ASSERT(adj_index0);
2723
2724           adj0 = ip_get_adjacency (lm, adj_index0);
2725
2726           ip0 = vlib_buffer_get_current (p0);
2727
2728           error0 = IP4_ERROR_NONE;
2729           next0 = 0;            /* drop on error */
2730
2731           /* Decrement TTL & update checksum. */
2732           if (! rewrite_for_locally_received_packets)
2733             {
2734               i32 ttl0 = ip0->ttl;
2735
2736               checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2737
2738               checksum0 += checksum0 >= 0xffff;
2739
2740               ip0->checksum = checksum0;
2741
2742               ASSERT (ip0->ttl > 0);
2743
2744               ttl0 -= 1;
2745
2746               ip0->ttl = ttl0;
2747
2748               ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2749
2750               error0 = ttl0 <= 0 ? IP4_ERROR_TIME_EXPIRED : error0;
2751             }
2752
2753           if (rewrite_for_locally_received_packets)
2754             {
2755               /*
2756                * If someone sends e.g. an icmp4 w/ src = dst = interface addr,
2757                * we end up here with a local adjacency in hand
2758                * The local adj rewrite data is 0xfefe on purpose.
2759                * Bad engineer, no donut for you.
2760                */
2761               if (PREDICT_FALSE(adj0->lookup_next_index
2762                                 == IP_LOOKUP_NEXT_LOCAL))
2763                 error0 = IP4_ERROR_SPOOFED_LOCAL_PACKETS;
2764               /*
2765                * We have to override the next_index in ARP adjacencies,
2766                * because they're set up for ip4-arp, not this node...
2767                */
2768               if (PREDICT_FALSE(adj0->lookup_next_index
2769                                 == IP_LOOKUP_NEXT_ARP))
2770                 next0_override = IP4_REWRITE_NEXT_ARP;
2771             }
2772
2773           /* Guess we are only writing on simple Ethernet header. */
2774           vnet_rewrite_one_header (adj0[0], ip0,
2775                                    sizeof (ethernet_header_t));
2776
2777           /* Update packet buffer attributes/set output interface. */
2778           rw_len0 = adj0[0].rewrite_header.data_bytes;
2779
2780           if (PREDICT_FALSE (rw_len0 > sizeof(ethernet_header_t)))
2781               vlib_increment_combined_counter
2782                   (&lm->adjacency_counters,
2783                    cpu_index, adj_index0,
2784                    /* packet increment */ 0,
2785                    /* byte increment */ rw_len0-sizeof(ethernet_header_t));
2786
2787           /* Check MTU of outgoing interface. */
2788           error0 = (vlib_buffer_length_in_chain (vm, p0)
2789                     > adj0[0].rewrite_header.max_l3_packet_bytes
2790                     ? IP4_ERROR_MTU_EXCEEDED
2791                     : error0);
2792
2793           p0->error = error_node->errors[error0];
2794           p0->current_data -= rw_len0;
2795           p0->current_length += rw_len0;
2796           vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2797             adj0[0].rewrite_header.sw_if_index;
2798
2799           next0 = (error0 == IP4_ERROR_NONE)
2800             ? adj0[0].rewrite_header.next_index : 0;
2801
2802           if (rewrite_for_locally_received_packets)
2803               next0 = next0 && next0_override ? next0_override : next0;
2804
2805           from += 1;
2806           n_left_from -= 1;
2807           to_next += 1;
2808           n_left_to_next -= 1;
2809
2810           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2811                                            to_next, n_left_to_next,
2812                                            pi0, next0);
2813         }
2814
2815       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2816     }
2817
2818   /* Need to do trace after rewrites to pick up new packet data. */
2819   if (node->flags & VLIB_NODE_FLAG_TRACE)
2820     ip4_forward_next_trace (vm, node, frame, adj_rx_tx);
2821
2822   return frame->n_vectors;
2823 }
2824
2825 static uword
2826 ip4_rewrite_transit (vlib_main_t * vm,
2827                      vlib_node_runtime_t * node,
2828                      vlib_frame_t * frame)
2829 {
2830   return ip4_rewrite_inline (vm, node, frame,
2831                              /* rewrite_for_locally_received_packets */ 0);
2832 }
2833
2834 static uword
2835 ip4_rewrite_local (vlib_main_t * vm,
2836                    vlib_node_runtime_t * node,
2837                    vlib_frame_t * frame)
2838 {
2839   return ip4_rewrite_inline (vm, node, frame,
2840                              /* rewrite_for_locally_received_packets */ 1);
2841 }
2842
2843 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
2844   .function = ip4_rewrite_transit,
2845   .name = "ip4-rewrite-transit",
2846   .vector_size = sizeof (u32),
2847
2848   .format_trace = format_ip4_rewrite_trace,
2849
2850   .n_next_nodes = 2,
2851   .next_nodes = {
2852     [IP4_REWRITE_NEXT_DROP] = "error-drop",
2853     [IP4_REWRITE_NEXT_ARP] = "ip4-arp",
2854   },
2855 };
2856
2857 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite_transit)
2858
2859 VLIB_REGISTER_NODE (ip4_rewrite_local_node) = {
2860   .function = ip4_rewrite_local,
2861   .name = "ip4-rewrite-local",
2862   .vector_size = sizeof (u32),
2863
2864   .sibling_of = "ip4-rewrite-transit",
2865
2866   .format_trace = format_ip4_rewrite_trace,
2867
2868   .n_next_nodes = 0,
2869 };
2870
2871 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_local_node, ip4_rewrite_local)
2872
2873 static clib_error_t *
2874 add_del_interface_table (vlib_main_t * vm,
2875                          unformat_input_t * input,
2876                          vlib_cli_command_t * cmd)
2877 {
2878   vnet_main_t * vnm = vnet_get_main();
2879   clib_error_t * error = 0;
2880   u32 sw_if_index, table_id;
2881
2882   sw_if_index = ~0;
2883
2884   if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2885     {
2886       error = clib_error_return (0, "unknown interface `%U'",
2887                                  format_unformat_error, input);
2888       goto done;
2889     }
2890
2891   if (unformat (input, "%d", &table_id))
2892     ;
2893   else
2894     {
2895       error = clib_error_return (0, "expected table id `%U'",
2896                                  format_unformat_error, input);
2897       goto done;
2898     }
2899
2900   {
2901     ip4_main_t * im = &ip4_main;
2902     ip4_fib_t * fib = find_ip4_fib_by_table_index_or_id (im, table_id, IP4_ROUTE_FLAG_TABLE_ID);
2903
2904     if (fib)
2905       {
2906         vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
2907         im->fib_index_by_sw_if_index[sw_if_index] = fib->index;
2908     }
2909   }
2910
2911  done:
2912   return error;
2913 }
2914
2915 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) = {
2916   .path = "set interface ip table",
2917   .function = add_del_interface_table,
2918   .short_help = "Add/delete FIB table id for interface",
2919 };
2920
2921
2922 static uword
2923 ip4_lookup_multicast (vlib_main_t * vm,
2924                       vlib_node_runtime_t * node,
2925                       vlib_frame_t * frame)
2926 {
2927   ip4_main_t * im = &ip4_main;
2928   ip_lookup_main_t * lm = &im->lookup_main;
2929   vlib_combined_counter_main_t * cm = &im->lookup_main.adjacency_counters;
2930   u32 n_left_from, n_left_to_next, * from, * to_next;
2931   ip_lookup_next_t next;
2932   u32 cpu_index = os_get_cpu_number();
2933
2934   from = vlib_frame_vector_args (frame);
2935   n_left_from = frame->n_vectors;
2936   next = node->cached_next_index;
2937
2938   while (n_left_from > 0)
2939     {
2940       vlib_get_next_frame (vm, node, next,
2941                            to_next, n_left_to_next);
2942
2943       while (n_left_from >= 4 && n_left_to_next >= 2)
2944         {
2945           vlib_buffer_t * p0, * p1;
2946           u32 pi0, pi1, adj_index0, adj_index1, wrong_next;
2947           ip_lookup_next_t next0, next1;
2948           ip4_header_t * ip0, * ip1;
2949           ip_adjacency_t * adj0, * adj1;
2950           u32 fib_index0, fib_index1;
2951           u32 flow_hash_config0, flow_hash_config1;
2952
2953           /* Prefetch next iteration. */
2954           {
2955             vlib_buffer_t * p2, * p3;
2956
2957             p2 = vlib_get_buffer (vm, from[2]);
2958             p3 = vlib_get_buffer (vm, from[3]);
2959
2960             vlib_prefetch_buffer_header (p2, LOAD);
2961             vlib_prefetch_buffer_header (p3, LOAD);
2962
2963             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
2964             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
2965           }
2966
2967           pi0 = to_next[0] = from[0];
2968           pi1 = to_next[1] = from[1];
2969
2970           p0 = vlib_get_buffer (vm, pi0);
2971           p1 = vlib_get_buffer (vm, pi1);
2972
2973           ip0 = vlib_buffer_get_current (p0);
2974           ip1 = vlib_buffer_get_current (p1);
2975
2976           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
2977           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
2978           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
2979             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
2980           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
2981             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
2982
2983           adj_index0 = ip4_fib_lookup_buffer (im, fib_index0,
2984                                               &ip0->dst_address, p0);
2985           adj_index1 = ip4_fib_lookup_buffer (im, fib_index1,
2986                                               &ip1->dst_address, p1);
2987
2988           adj0 = ip_get_adjacency (lm, adj_index0);
2989           adj1 = ip_get_adjacency (lm, adj_index1);
2990
2991           next0 = adj0->lookup_next_index;
2992           next1 = adj1->lookup_next_index;
2993
2994           flow_hash_config0 =
2995               vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
2996
2997           flow_hash_config1 =
2998               vec_elt_at_index (im->fibs, fib_index1)->flow_hash_config;
2999
3000           vnet_buffer (p0)->ip.flow_hash = ip4_compute_flow_hash
3001               (ip0, flow_hash_config0);
3002
3003           vnet_buffer (p1)->ip.flow_hash = ip4_compute_flow_hash
3004               (ip1, flow_hash_config1);
3005
3006           ASSERT (adj0->n_adj > 0);
3007           ASSERT (adj1->n_adj > 0);
3008           ASSERT (is_pow2 (adj0->n_adj));
3009           ASSERT (is_pow2 (adj1->n_adj));
3010           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
3011           adj_index1 += (vnet_buffer (p1)->ip.flow_hash & (adj1->n_adj - 1));
3012
3013           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
3014           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
3015
3016           if (1) /* $$$$$$ HACK FIXME */
3017           vlib_increment_combined_counter
3018               (cm, cpu_index, adj_index0, 1,
3019                vlib_buffer_length_in_chain (vm, p0));
3020           if (1) /* $$$$$$ HACK FIXME */
3021           vlib_increment_combined_counter
3022               (cm, cpu_index, adj_index1, 1,
3023                vlib_buffer_length_in_chain (vm, p1));
3024
3025           from += 2;
3026           to_next += 2;
3027           n_left_to_next -= 2;
3028           n_left_from -= 2;
3029
3030           wrong_next = (next0 != next) + 2*(next1 != next);
3031           if (PREDICT_FALSE (wrong_next != 0))
3032             {
3033               switch (wrong_next)
3034                 {
3035                 case 1:
3036                   /* A B A */
3037                   to_next[-2] = pi1;
3038                   to_next -= 1;
3039                   n_left_to_next += 1;
3040                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
3041                   break;
3042
3043                 case 2:
3044                   /* A A B */
3045                   to_next -= 1;
3046                   n_left_to_next += 1;
3047                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
3048                   break;
3049
3050                 case 3:
3051                   /* A B C */
3052                   to_next -= 2;
3053                   n_left_to_next += 2;
3054                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
3055                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
3056                   if (next0 == next1)
3057                     {
3058                       /* A B B */
3059                       vlib_put_next_frame (vm, node, next, n_left_to_next);
3060                       next = next1;
3061                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
3062                     }
3063                 }
3064             }
3065         }
3066
3067       while (n_left_from > 0 && n_left_to_next > 0)
3068         {
3069           vlib_buffer_t * p0;
3070           ip4_header_t * ip0;
3071           u32 pi0, adj_index0;
3072           ip_lookup_next_t next0;
3073           ip_adjacency_t * adj0;
3074           u32 fib_index0;
3075           u32 flow_hash_config0;
3076
3077           pi0 = from[0];
3078           to_next[0] = pi0;
3079
3080           p0 = vlib_get_buffer (vm, pi0);
3081
3082           ip0 = vlib_buffer_get_current (p0);
3083
3084           fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
3085                                 vnet_buffer (p0)->sw_if_index[VLIB_RX]);
3086           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
3087               fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
3088
3089           adj_index0 = ip4_fib_lookup_buffer (im, fib_index0,
3090                                               &ip0->dst_address, p0);
3091
3092           adj0 = ip_get_adjacency (lm, adj_index0);
3093
3094           next0 = adj0->lookup_next_index;
3095
3096           flow_hash_config0 =
3097               vec_elt_at_index (im->fibs, fib_index0)->flow_hash_config;
3098
3099           vnet_buffer (p0)->ip.flow_hash =
3100             ip4_compute_flow_hash (ip0, flow_hash_config0);
3101
3102           ASSERT (adj0->n_adj > 0);
3103           ASSERT (is_pow2 (adj0->n_adj));
3104           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
3105
3106           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
3107
3108           if (1) /* $$$$$$ HACK FIXME */
3109               vlib_increment_combined_counter
3110                   (cm, cpu_index, adj_index0, 1,
3111                    vlib_buffer_length_in_chain (vm, p0));
3112
3113           from += 1;
3114           to_next += 1;
3115           n_left_to_next -= 1;
3116           n_left_from -= 1;
3117
3118           if (PREDICT_FALSE (next0 != next))
3119             {
3120               n_left_to_next += 1;
3121               vlib_put_next_frame (vm, node, next, n_left_to_next);
3122               next = next0;
3123               vlib_get_next_frame (vm, node, next,
3124                                    to_next, n_left_to_next);
3125               to_next[0] = pi0;
3126               to_next += 1;
3127               n_left_to_next -= 1;
3128             }
3129         }
3130
3131       vlib_put_next_frame (vm, node, next, n_left_to_next);
3132     }
3133
3134   if (node->flags & VLIB_NODE_FLAG_TRACE)
3135       ip4_forward_next_trace(vm, node, frame, VLIB_TX);
3136
3137   return frame->n_vectors;
3138 }
3139
3140 VLIB_REGISTER_NODE (ip4_lookup_multicast_node,static) = {
3141   .function = ip4_lookup_multicast,
3142   .name = "ip4-lookup-multicast",
3143   .vector_size = sizeof (u32),
3144   .sibling_of = "ip4-lookup",
3145   .format_trace = format_ip4_lookup_trace,
3146
3147   .n_next_nodes = 0,
3148 };
3149
3150 VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_multicast_node, ip4_lookup_multicast)
3151
3152 VLIB_REGISTER_NODE (ip4_multicast_node,static) = {
3153   .function = ip4_drop,
3154   .name = "ip4-multicast",
3155   .vector_size = sizeof (u32),
3156
3157   .format_trace = format_ip4_forward_next_trace,
3158
3159   .n_next_nodes = 1,
3160   .next_nodes = {
3161     [0] = "error-drop",
3162   },
3163 };
3164
3165 int ip4_lookup_validate (ip4_address_t *a, u32 fib_index0)
3166 {
3167   ip4_main_t * im = &ip4_main;
3168   ip4_fib_mtrie_t * mtrie0;
3169   ip4_fib_mtrie_leaf_t leaf0;
3170   u32 adj_index0;
3171
3172   mtrie0 = &vec_elt_at_index (im->fibs, fib_index0)->mtrie;
3173
3174   leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
3175   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 0);
3176   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 1);
3177   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
3178   leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
3179
3180   /* Handle default route. */
3181   leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
3182
3183   adj_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
3184
3185   return adj_index0 == ip4_fib_lookup_with_table (im, fib_index0,
3186                                                   a,
3187                                                   /* no_default_route */ 0);
3188 }
3189
3190 static clib_error_t *
3191 test_lookup_command_fn (vlib_main_t * vm,
3192                         unformat_input_t * input,
3193                         vlib_cli_command_t * cmd)
3194 {
3195   u32 table_id = 0;
3196   f64 count = 1;
3197   u32 n;
3198   int i;
3199   ip4_address_t ip4_base_address;
3200   u64 errors = 0;
3201
3202   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3203       if (unformat (input, "table %d", &table_id))
3204         ;
3205       else if (unformat (input, "count %f", &count))
3206         ;
3207
3208       else if (unformat (input, "%U",
3209                          unformat_ip4_address, &ip4_base_address))
3210         ;
3211       else
3212         return clib_error_return (0, "unknown input `%U'",
3213                                   format_unformat_error, input);
3214   }
3215
3216   n = count;
3217
3218   for (i = 0; i < n; i++)
3219     {
3220       if (!ip4_lookup_validate (&ip4_base_address, table_id))
3221         errors++;
3222
3223       ip4_base_address.as_u32 =
3224         clib_host_to_net_u32 (1 +
3225                               clib_net_to_host_u32 (ip4_base_address.as_u32));
3226     }
3227
3228   if (errors)
3229     vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
3230   else
3231     vlib_cli_output (vm, "No errors in %d lookups\n", n);
3232
3233   return 0;
3234 }
3235
3236 VLIB_CLI_COMMAND (lookup_test_command, static) = {
3237     .path = "test lookup",
3238     .short_help = "test lookup",
3239     .function = test_lookup_command_fn,
3240 };
3241
3242 int vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
3243 {
3244   ip4_main_t * im4 = &ip4_main;
3245   ip4_fib_t * fib;
3246   uword * p = hash_get (im4->fib_index_by_table_id, table_id);
3247
3248   if (p == 0)
3249     return VNET_API_ERROR_NO_SUCH_FIB;
3250
3251   fib = vec_elt_at_index (im4->fibs, p[0]);
3252
3253   fib->flow_hash_config = flow_hash_config;
3254   return 0;
3255 }
3256
3257 static clib_error_t *
3258 set_ip_flow_hash_command_fn (vlib_main_t * vm,
3259                              unformat_input_t * input,
3260                              vlib_cli_command_t * cmd)
3261 {
3262   int matched = 0;
3263   u32 table_id = 0;
3264   u32 flow_hash_config = 0;
3265   int rv;
3266
3267   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3268     if (unformat (input, "table %d", &table_id))
3269       matched = 1;
3270 #define _(a,v) \
3271     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
3272     foreach_flow_hash_bit
3273 #undef _
3274     else break;
3275   }
3276
3277   if (matched == 0)
3278     return clib_error_return (0, "unknown input `%U'",
3279                               format_unformat_error, input);
3280
3281   rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
3282   switch (rv)
3283     {
3284     case 0:
3285       break;
3286
3287     case VNET_API_ERROR_NO_SUCH_FIB:
3288       return clib_error_return (0, "no such FIB table %d", table_id);
3289
3290     default:
3291       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3292       break;
3293     }
3294
3295   return 0;
3296 }
3297
3298 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) = {
3299   .path = "set ip flow-hash",
3300   .short_help =
3301   "set ip table flow-hash table <fib-id> src dst sport dport proto reverse",
3302   .function = set_ip_flow_hash_command_fn,
3303 };
3304
3305 int vnet_set_ip4_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3306                                  u32 table_index)
3307 {
3308   vnet_main_t * vnm = vnet_get_main();
3309   vnet_interface_main_t * im = &vnm->interface_main;
3310   ip4_main_t * ipm = &ip4_main;
3311   ip_lookup_main_t * lm = &ipm->lookup_main;
3312   vnet_classify_main_t * cm = &vnet_classify_main;
3313
3314   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3315     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3316
3317   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3318     return VNET_API_ERROR_NO_SUCH_ENTRY;
3319
3320   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3321   lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
3322
3323   return 0;
3324 }
3325
3326 static clib_error_t *
3327 set_ip_classify_command_fn (vlib_main_t * vm,
3328                             unformat_input_t * input,
3329                             vlib_cli_command_t * cmd)
3330 {
3331   u32 table_index = ~0;
3332   int table_index_set = 0;
3333   u32 sw_if_index = ~0;
3334   int rv;
3335
3336   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3337     if (unformat (input, "table-index %d", &table_index))
3338       table_index_set = 1;
3339     else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3340                        vnet_get_main(), &sw_if_index))
3341       ;
3342     else
3343       break;
3344   }
3345
3346   if (table_index_set == 0)
3347     return clib_error_return (0, "classify table-index must be specified");
3348
3349   if (sw_if_index == ~0)
3350     return clib_error_return (0, "interface / subif must be specified");
3351
3352   rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
3353
3354   switch (rv)
3355     {
3356     case 0:
3357       break;
3358
3359     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3360       return clib_error_return (0, "No such interface");
3361
3362     case VNET_API_ERROR_NO_SUCH_ENTRY:
3363       return clib_error_return (0, "No such classifier table");
3364     }
3365   return 0;
3366 }
3367
3368 VLIB_CLI_COMMAND (set_ip_classify_command, static) = {
3369     .path = "set ip classify",
3370     .short_help =
3371     "set ip classify intfc <int> table-index <index>",
3372     .function = set_ip_classify_command_fn,
3373 };
3374