vnet/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h> /* for ethernet_header_t */
  43 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  44 #include <vppinfra/cache.h>
  45
  46 #include <vppinfra/bihash_template.c>
  47
  48 static void compute_prefix_lengths_in_search_order (ip6_main_t * im)
  49 {
  50   int i;
  51   vec_reset_length (im->prefix_lengths_in_search_order);
  52   /* Note: bitmap reversed so this is in fact a longest prefix match */
  53   clib_bitmap_foreach (i, im->non_empty_dst_address_length_bitmap,
  54   ({
  55     int dst_address_length = 128 - i;
  56     vec_add1 (im->prefix_lengths_in_search_order, dst_address_length);
  57   }));
  58 }
  59
  60 u32
  61 ip6_fib_lookup_with_table (ip6_main_t * im, u32 fib_index, ip6_address_t * dst)
  62 {
  63   ip_lookup_main_t * lm = &im->lookup_main;
  64   int i, len;
  65   int rv;
  66   BVT(clib_bihash_kv) kv, value;
  67   u64 fib;
  68
  69   len = vec_len (im->prefix_lengths_in_search_order);
  70
  71   kv.key[0] = dst->as_u64[0];
  72   kv.key[1] = dst->as_u64[1];
  73   fib = ((u64)((fib_index))<<32);
  74
  75   for (i = 0; i < len; i++)
  76     {
  77       int dst_address_length = im->prefix_lengths_in_search_order[i];
  78       ip6_address_t * mask = &im->fib_masks[dst_address_length];
  79
  80       ASSERT(dst_address_length >= 0 && dst_address_length <= 128);
  81       //As lengths are decreasing, masks are increasingly specific.
  82       kv.key[0] &= mask->as_u64[0];
  83       kv.key[1] &= mask->as_u64[1];
  84       kv.key[2] = fib | dst_address_length;
  85
  86       rv = BV(clib_bihash_search_inline_2)(&im->ip6_lookup_table, &kv, &value);
  87       if (rv == 0)
  88         return value.value;
  89     }
  90
  91   return lm->miss_adj_index;
  92 }
  93
  94 u32 ip6_fib_lookup (ip6_main_t * im, u32 sw_if_index, ip6_address_t * dst)
  95 {
  96     u32 fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
  97     return ip6_fib_lookup_with_table (im, fib_index, dst);
  98 }
  99
 100 void
 101 vnet_ip6_fib_init (ip6_main_t * im, u32 fib_index)
 102 {
 103   ip_lookup_main_t * lm = &im->lookup_main;
 104   ip6_add_del_route_args_t a;
 105   ip_adjacency_t * adj;
 106
 107   memset(&a, 0x0, sizeof(ip6_add_del_route_args_t));
 108
 109   a.table_index_or_table_id = fib_index;
 110   a.flags = (IP6_ROUTE_FLAG_ADD
 111              | IP6_ROUTE_FLAG_FIB_INDEX
 112              | IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 113              | IP6_ROUTE_FLAG_NO_REDISTRIBUTE);
 114
 115   /* Add ff02::1:ff00:0/104 via local route for all tables.
 116      This is required for neighbor discovery to work. */
 117   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 118                           &a.adj_index);
 119   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 120   adj->if_address_index = ~0;
 121   adj->rewrite_header.data_bytes = 0;
 122
 123   ip6_set_solicited_node_multicast_address (&a.dst_address, 0);
 124
 125   a.dst_address_length = 104;
 126   ip6_add_del_route (im, &a);
 127
 128   /* Add all-routers multicast address via local route for all tables */
 129   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 130                           &a.adj_index);
 131   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 132   adj->if_address_index = ~0;
 133   adj->rewrite_header.data_bytes = 0;
 134
 135   ip6_set_reserved_multicast_address (&a.dst_address,
 136                                       IP6_MULTICAST_SCOPE_link_local,
 137                                       IP6_MULTICAST_GROUP_ID_all_routers);
 138
 139   a.dst_address_length = 128;
 140   ip6_add_del_route (im, &a);
 141
 142   /* Add all-nodes multicast address via local route for all tables */
 143   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 144                           &a.adj_index);
 145   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 146   adj->if_address_index = ~0;
 147   adj->rewrite_header.data_bytes = 0;
 148
 149   ip6_set_reserved_multicast_address (&a.dst_address,
 150                                       IP6_MULTICAST_SCOPE_link_local,
 151                                       IP6_MULTICAST_GROUP_ID_all_hosts);
 152
 153   a.dst_address_length = 128;
 154   ip6_add_del_route (im, &a);
 155
 156   /* Add all-mldv2  multicast address via local route for all tables */
 157   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 158                           &a.adj_index);
 159   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 160   adj->if_address_index = ~0;
 161   adj->rewrite_header.data_bytes = 0;
 162
 163   ip6_set_reserved_multicast_address (&a.dst_address,
 164                                       IP6_MULTICAST_SCOPE_link_local,
 165                                       IP6_MULTICAST_GROUP_ID_mldv2_routers);
 166
 167   a.dst_address_length = 128;
 168   ip6_add_del_route (im, &a);
 169 }
 170
 171 static ip6_fib_t *
 172 create_fib_with_table_id (ip6_main_t * im, u32 table_id)
 173 {
 174   ip6_fib_t * fib;
 175   hash_set (im->fib_index_by_table_id, table_id, vec_len (im->fibs));
 176   vec_add2 (im->fibs, fib, 1);
 177   fib->table_id = table_id;
 178   fib->index = fib - im->fibs;
 179   fib->flow_hash_config = IP_FLOW_HASH_DEFAULT;
 180   vnet_ip6_fib_init (im, fib->index);
 181   return fib;
 182 }
 183
 184 ip6_fib_t *
 185 find_ip6_fib_by_table_index_or_id (ip6_main_t * im, u32 table_index_or_id, u32 flags)
 186 {
 187   uword * p, fib_index;
 188
 189   fib_index = table_index_or_id;
 190   if (! (flags & IP6_ROUTE_FLAG_FIB_INDEX))
 191     {
 192       p = hash_get (im->fib_index_by_table_id, table_index_or_id);
 193       if (! p)
 194         return create_fib_with_table_id (im, table_index_or_id);
 195       fib_index = p[0];
 196     }
 197   return vec_elt_at_index (im->fibs, fib_index);
 198 }
 199
 200 void ip6_add_del_route (ip6_main_t * im, ip6_add_del_route_args_t * a)
 201 {
 202   ip_lookup_main_t * lm = &im->lookup_main;
 203   ip6_fib_t * fib;
 204   ip6_address_t dst_address;
 205   u32 dst_address_length, adj_index;
 206   uword is_del;
 207   u32 old_adj_index = ~0;
 208   BVT(clib_bihash_kv) kv, value;
 209
 210   vlib_smp_unsafe_warning();
 211
 212   is_del = (a->flags & IP6_ROUTE_FLAG_DEL) != 0;
 213
 214   /* Either create new adjacency or use given one depending on arguments. */
 215   if (a->n_add_adj > 0)
 216     {
 217       ip_add_adjacency (lm, a->add_adj, a->n_add_adj, &adj_index);
 218       ip_call_add_del_adjacency_callbacks (lm, adj_index, /* is_del */ 0);
 219     }
 220   else
 221     adj_index = a->adj_index;
 222
 223   dst_address = a->dst_address;
 224   dst_address_length = a->dst_address_length;
 225   fib = find_ip6_fib_by_table_index_or_id (im, a->table_index_or_table_id,
 226                                            a->flags);
 227
 228   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 229   ip6_address_mask (&dst_address, &im->fib_masks[dst_address_length]);
 230
 231   /* refcount accounting */
 232   if (is_del)
 233     {
 234       ASSERT (im->dst_address_length_refcounts[dst_address_length] > 0);
 235       if (--im->dst_address_length_refcounts[dst_address_length] == 0)
 236         {
 237           im->non_empty_dst_address_length_bitmap =
 238             clib_bitmap_set (im->non_empty_dst_address_length_bitmap,
 239                              128 - dst_address_length, 0);
 240           compute_prefix_lengths_in_search_order (im);
 241         }
 242     }
 243   else
 244     {
 245       im->dst_address_length_refcounts[dst_address_length]++;
 246
 247       im->non_empty_dst_address_length_bitmap =
 248         clib_bitmap_set (im->non_empty_dst_address_length_bitmap,
 249                              128 - dst_address_length, 1);
 250       compute_prefix_lengths_in_search_order (im);
 251     }
 252
 253   kv.key[0] = dst_address.as_u64[0];
 254   kv.key[1] = dst_address.as_u64[1];
 255   kv.key[2] = ((u64)((fib - im->fibs))<<32) | dst_address_length;
 256
 257   if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) == 0)
 258     old_adj_index = value.value;
 259
 260   if (is_del)
 261     BV(clib_bihash_add_del) (&im->ip6_lookup_table, &kv, 0 /* is_add */);
 262   else
 263     {
 264       /* Make sure adj index is valid. */
 265       if (CLIB_DEBUG > 0)
 266         (void) ip_get_adjacency (lm, adj_index);
 267
 268       kv.value = adj_index;
 269
 270       BV(clib_bihash_add_del) (&im->ip6_lookup_table, &kv, 1 /* is_add */);
 271     }
 272
 273   /* Avoid spurious reference count increments */
 274   if (old_adj_index == adj_index && !(a->flags & IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY))
 275     {
 276       ip_adjacency_t * adj = ip_get_adjacency (lm, adj_index);
 277       if (adj->share_count > 0)
 278         adj->share_count --;
 279     }
 280
 281   /* Delete old adjacency index if present and changed. */
 282   {
 283     if (! (a->flags & IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY)
 284         && old_adj_index != ~0
 285         && old_adj_index != adj_index)
 286       ip_del_adjacency (lm, old_adj_index);
 287   }
 288 }
 289
 290 void
 291 ip6_add_del_route_next_hop (ip6_main_t * im,
 292                             u32 flags,
 293                             ip6_address_t * dst_address,
 294                             u32 dst_address_length,
 295                             ip6_address_t * next_hop,
 296                             u32 next_hop_sw_if_index,
 297                             u32 next_hop_weight, u32 adj_index,
 298                             u32 explicit_fib_index)
 299 {
 300   vnet_main_t * vnm = vnet_get_main();
 301   ip_lookup_main_t * lm = &im->lookup_main;
 302   u32 fib_index;
 303   ip6_fib_t * fib;
 304   ip6_address_t masked_dst_address;
 305   u32 old_mp_adj_index, new_mp_adj_index;
 306   u32 dst_adj_index, nh_adj_index;
 307   int rv;
 308   ip_adjacency_t * dst_adj;
 309   ip_multipath_adjacency_t * old_mp, * new_mp;
 310   int is_del = (flags & IP6_ROUTE_FLAG_DEL) != 0;
 311   int is_interface_next_hop;
 312   clib_error_t * error = 0;
 313   uword * nh_result;
 314   BVT(clib_bihash_kv) kv, value;
 315
 316   vlib_smp_unsafe_warning();
 317
 318   if (explicit_fib_index == (u32)~0)
 319     fib_index = vec_elt (im->fib_index_by_sw_if_index, next_hop_sw_if_index);
 320   else
 321     fib_index = explicit_fib_index;
 322
 323   fib = vec_elt_at_index (im->fibs, fib_index);
 324
 325   /* Lookup next hop to be added or deleted. */
 326   is_interface_next_hop = ip6_address_is_zero (next_hop);
 327   if (adj_index == (u32)~0)
 328     {
 329       if (is_interface_next_hop)
 330         {
 331           nh_result = hash_get (im->interface_route_adj_index_by_sw_if_index,
 332                                 next_hop_sw_if_index);
 333           if (nh_result)
 334             nh_adj_index = *nh_result;
 335           else
 336             {
 337               ip_adjacency_t * adj;
 338               adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 339                                       &nh_adj_index);
 340               ip6_adjacency_set_interface_route (vnm, adj,
 341                                                  next_hop_sw_if_index, ~0);
 342               ip_call_add_del_adjacency_callbacks
 343                 (lm, next_hop_sw_if_index, /* is_del */ 0);
 344               hash_set (im->interface_route_adj_index_by_sw_if_index,
 345                         next_hop_sw_if_index, nh_adj_index);
 346             }
 347         }
 348       else
 349         {
 350           /* Look for the interface /128 route */
 351           kv.key[0] = next_hop->as_u64[0];
 352           kv.key[1] = next_hop->as_u64[1];
 353           kv.key[2] = ((u64)((fib - im->fibs))<<32) | 128;
 354
 355           if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) < 0)
 356           {
 357             ip_adjacency_t * adj;
 358             nh_adj_index = ip6_fib_lookup_with_table (im, fib_index, next_hop);
 359             adj = ip_get_adjacency (lm, nh_adj_index);
 360             /* if ND interface adjacencty is present, we need to
 361                              install ND adjaceny for specific next hop */
 362             if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP &&
 363                 adj->arp.next_hop.ip6.as_u64[0] == 0 &&
 364                 adj->arp.next_hop.ip6.as_u64[1] == 0)
 365             {
 366               nh_adj_index = vnet_ip6_neighbor_glean_add(fib_index, next_hop);
 367             }
 368             else
 369             {
 370               vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
 371               error = clib_error_return (0, "next-hop %U/128 not in FIB",
 372                                          format_ip6_address, next_hop);
 373               goto done;
 374             }
 375           }
 376           else
 377             nh_adj_index = value.value;
 378
 379         }
 380     }
 381   else
 382     {
 383       /* Look for the interface /128 route */
 384       kv.key[0] = next_hop->as_u64[0];
 385       kv.key[1] = next_hop->as_u64[1];
 386       kv.key[2] = ((u64)((fib - im->fibs))<<32) | 128;
 387
 388       if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) < 0)
 389         {
 390           vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
 391           error = clib_error_return (0, "next-hop %U/128 not in FIB",
 392                                      format_ip6_address, next_hop);
 393           goto done;
 394         }
 395
 396       nh_adj_index = value.value;
 397     }
 398
 399   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 400   masked_dst_address = dst_address[0];
 401   ip6_address_mask (&masked_dst_address, &im->fib_masks[dst_address_length]);
 402
 403   kv.key[0] = masked_dst_address.as_u64[0];
 404   kv.key[1] = masked_dst_address.as_u64[1];
 405   kv.key[2] = ((u64)((fib - im->fibs))<<32) | dst_address_length;
 406
 407   rv = BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value);
 408
 409   if (rv == 0)
 410     {
 411       dst_adj_index = value.value;
 412       dst_adj = ip_get_adjacency (lm, dst_adj_index);
 413     }
 414   else
 415     {
 416       /* For deletes destination must be known. */
 417       if (is_del)
 418         {
 419           vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
 420           error = clib_error_return (0, "unknown destination %U/%d",
 421                                      format_ip6_address, dst_address,
 422                                      dst_address_length);
 423           goto done;
 424         }
 425
 426       dst_adj_index = ~0;
 427       dst_adj = 0;
 428     }
 429
 430   /* Ignore adds of X/128 with next hop of X. */
 431   if (! is_del
 432       && dst_address_length == 128
 433       && ip6_address_is_equal (dst_address, next_hop))
 434     {
 435       vnm->api_errno = VNET_API_ERROR_PREFIX_MATCHES_NEXT_HOP;
 436       error = clib_error_return (0, "prefix matches next hop %U/%d",
 437                                  format_ip6_address, dst_address,
 438                                  dst_address_length);
 439       goto done;
 440     }
 441
 442   /* Destination is not known and default weight is set so add route
 443      to existing non-multipath adjacency */
 444   if (dst_adj_index == ~0 && next_hop_weight == 1 && next_hop_sw_if_index == ~0)
 445   {
 446     /* create new adjacency */
 447     ip6_add_del_route_args_t a;
 448     a.table_index_or_table_id = fib_index;
 449     a.flags = ((is_del ? IP6_ROUTE_FLAG_DEL : IP6_ROUTE_FLAG_ADD)
 450         | IP6_ROUTE_FLAG_FIB_INDEX
 451         | IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 452         | (flags & (IP6_ROUTE_FLAG_NO_REDISTRIBUTE
 453             | IP6_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
 454     a.dst_address = dst_address[0];
 455     a.dst_address_length = dst_address_length;
 456     a.adj_index = nh_adj_index;
 457     a.add_adj = 0;
 458     a.n_add_adj = 0;
 459
 460     ip6_add_del_route (im, &a);
 461     goto done;
 462   }
 463
 464   old_mp_adj_index = dst_adj ? dst_adj->heap_handle : ~0;
 465
 466   if (! ip_multipath_adjacency_add_del_next_hop
 467       (lm, is_del,
 468        dst_adj ? dst_adj->heap_handle : ~0,
 469        nh_adj_index,
 470        next_hop_weight,
 471        &new_mp_adj_index))
 472     {
 473       vnm->api_errno = VNET_API_ERROR_NEXT_HOP_NOT_FOUND_MP;
 474       error = clib_error_return
 475         (0, "requested deleting next-hop %U not found in multi-path",
 476          format_ip6_address, next_hop);
 477       goto done;
 478     }
 479
 480   old_mp = new_mp = 0;
 481   if (old_mp_adj_index != ~0)
 482     old_mp = vec_elt_at_index (lm->multipath_adjacencies, old_mp_adj_index);
 483   if (new_mp_adj_index != ~0)
 484     new_mp = vec_elt_at_index (lm->multipath_adjacencies, new_mp_adj_index);
 485
 486   if (old_mp != new_mp)
 487     {
 488       ip6_add_del_route_args_t a;
 489       a.table_index_or_table_id = fib_index;
 490       a.flags = ((is_del ? IP6_ROUTE_FLAG_DEL : IP6_ROUTE_FLAG_ADD)
 491                  | IP6_ROUTE_FLAG_FIB_INDEX
 492                  | IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 493                  | (flags & IP6_ROUTE_FLAG_NO_REDISTRIBUTE));
 494       a.dst_address = dst_address[0];
 495       a.dst_address_length = dst_address_length;
 496       a.adj_index = new_mp ? new_mp->adj_index : dst_adj_index;
 497       a.add_adj = 0;
 498       a.n_add_adj = 0;
 499
 500       ip6_add_del_route (im, &a);
 501     }
 502
 503  done:
 504   if (error)
 505     clib_error_report (error);
 506 }
 507
 508 u32
 509 ip6_get_route (ip6_main_t * im,
 510                u32 table_index_or_table_id,
 511                u32 flags,
 512                ip6_address_t * address,
 513                u32 address_length)
 514 {
 515   ip6_fib_t * fib = find_ip6_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 516   ip6_address_t masked_address;
 517   BVT(clib_bihash_kv) kv, value;
 518
 519   ASSERT (address_length < ARRAY_LEN (im->fib_masks));
 520   clib_memcpy (&masked_address, address, sizeof (masked_address));
 521   ip6_address_mask (&masked_address, &im->fib_masks[address_length]);
 522
 523   kv.key[0] = masked_address.as_u64[0];
 524   kv.key[1] = masked_address.as_u64[1];
 525   kv.key[2] = ((u64)((fib - im->fibs))<<32) | address_length;
 526
 527   if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) == 0)
 528     return (value.value);
 529   return 0;
 530 }
 531
 532 void
 533 ip6_foreach_matching_route (ip6_main_t * im,
 534                             u32 table_index_or_table_id,
 535                             u32 flags,
 536                             ip6_address_t * dst_address,
 537                             u32 address_length,
 538                             ip6_address_t ** results,
 539                             u8 ** result_lengths)
 540 {
 541   ip6_fib_t * fib =
 542     find_ip6_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 543   BVT(clib_bihash) * h = &im->ip6_lookup_table;
 544   BVT(clib_bihash_value) * v;
 545   clib_bihash_bucket_t * b;
 546   int i, j, k;
 547
 548   if (*results)
 549     _vec_len (*results) = 0;
 550   if (*result_lengths)
 551     _vec_len (*result_lengths) = 0;
 552
 553   /* Walk the table looking for routes which match the supplied address */
 554   for (i = 0; i < h->nbuckets; i++)
 555     {
 556       b = &h->buckets [i];
 557       if (b->offset == 0)
 558           continue;
 559
 560       v = BV(clib_bihash_get_value) (h, b->offset);
 561       for (j = 0; j < (1<<b->log2_pages); j++)
 562         {
 563           for (k = 0; k < BIHASH_KVP_PER_PAGE; k++)
 564             {
 565               if (BV(clib_bihash_is_free)(&v->kvp[k]))
 566                 continue;
 567
 568               if ((v->kvp[k].key[2]
 569                    == (((u64)((fib - im->fibs))<<32) | address_length))
 570                   && ip6_destination_matches_route
 571                   (im, dst_address, (ip6_address_t *) &v->kvp[k],
 572                    address_length))
 573                 {
 574                   ip6_address_t * a;
 575
 576                   a = (ip6_address_t *)(&v->kvp[k]);
 577
 578                   vec_add1 (*results, a[0]);
 579                   vec_add1 (*result_lengths, address_length);
 580                 }
 581             }
 582           v++;
 583         }
 584     }
 585 }
 586
 587 void ip6_maybe_remap_adjacencies (ip6_main_t * im,
 588                                   u32 table_index_or_table_id,
 589                                   u32 flags)
 590 {
 591 #if SOONE
 592   ip6_fib_t * fib
 593     = find_ip6_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 594 #endif
 595   ip_lookup_main_t * lm = &im->lookup_main;
 596
 597   if (lm->n_adjacency_remaps == 0)
 598     return;
 599
 600   clib_warning ("unimplemented, please report to vpp-dev@cisco.com");
 601
 602   /* All remaps have been performed. */
 603   lm->n_adjacency_remaps = 0;
 604 }
 605
 606 void ip6_delete_matching_routes (ip6_main_t * im,
 607                                  u32 table_index_or_table_id,
 608                                  u32 flags,
 609                                  ip6_address_t * address,
 610                                  u32 address_length)
 611 {
 612   /* $$$$ static may be OK - this should happen only on thread 0 */
 613   static ip6_address_t * matching_addresses;
 614   static u8 * matching_address_lengths;
 615   u32 l, i;
 616   ip6_add_del_route_args_t a;
 617
 618   vlib_smp_unsafe_warning();
 619
 620   a.flags = IP6_ROUTE_FLAG_DEL | IP6_ROUTE_FLAG_NO_REDISTRIBUTE | flags;
 621   a.table_index_or_table_id = table_index_or_table_id;
 622   a.adj_index = ~0;
 623   a.add_adj = 0;
 624   a.n_add_adj = 0;
 625
 626   for (l = address_length + 1; l <= 128; l++)
 627     {
 628       ip6_foreach_matching_route (im, table_index_or_table_id, flags,
 629                                   address,
 630                                   l,
 631                                   &matching_addresses,
 632                                   &matching_address_lengths);
 633       for (i = 0; i < vec_len (matching_addresses); i++)
 634         {
 635           a.dst_address = matching_addresses[i];
 636           a.dst_address_length = matching_address_lengths[i];
 637           ip6_add_del_route (im, &a);
 638         }
 639     }
 640
 641   ip6_maybe_remap_adjacencies (im, table_index_or_table_id, flags);
 642 }
 643
 644 static uword
 645 ip6_lookup (vlib_main_t * vm,
 646             vlib_node_runtime_t * node,
 647             vlib_frame_t * frame)
 648 {
 649   ip6_main_t * im = &ip6_main;
 650   ip_lookup_main_t * lm = &im->lookup_main;
 651   vlib_combined_counter_main_t * cm = &im->lookup_main.adjacency_counters;
 652   u32 n_left_from, n_left_to_next, * from, * to_next;
 653   ip_lookup_next_t next;
 654   u32 cpu_index = os_get_cpu_number();
 655
 656   from = vlib_frame_vector_args (frame);
 657   n_left_from = frame->n_vectors;
 658   next = node->cached_next_index;
 659
 660   while (n_left_from > 0)
 661     {
 662       vlib_get_next_frame (vm, node, next,
 663                            to_next, n_left_to_next);
 664
 665       while (n_left_from >= 4 && n_left_to_next >= 2)
 666         {
 667           vlib_buffer_t * p0, * p1;
 668           u32 pi0, pi1, adj_index0, adj_index1, wrong_next;
 669           ip_lookup_next_t next0, next1;
 670           ip6_header_t * ip0, * ip1;
 671           ip_adjacency_t * adj0, * adj1;
 672           u32 fib_index0, fib_index1;
 673           u32 flow_hash_config0, flow_hash_config1;
 674
 675           /* Prefetch next iteration. */
 676           {
 677             vlib_buffer_t * p2, * p3;
 678
 679             p2 = vlib_get_buffer (vm, from[2]);
 680             p3 = vlib_get_buffer (vm, from[3]);
 681
 682             vlib_prefetch_buffer_header (p2, LOAD);
 683             vlib_prefetch_buffer_header (p3, LOAD);
 684             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
 685             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
 686           }
 687
 688           pi0 = to_next[0] = from[0];
 689           pi1 = to_next[1] = from[1];
 690
 691           p0 = vlib_get_buffer (vm, pi0);
 692           p1 = vlib_get_buffer (vm, pi1);
 693
 694           ip0 = vlib_buffer_get_current (p0);
 695           ip1 = vlib_buffer_get_current (p1);
 696
 697           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 698           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 699
 700           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 701             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 702           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
 703             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
 704
 705           adj_index0 = ip6_fib_lookup_with_table (im, fib_index0,
 706                                                   &ip0->dst_address);
 707           adj_index1 = ip6_fib_lookup_with_table (im, fib_index1,
 708                                                   &ip1->dst_address);
 709
 710           adj0 = ip_get_adjacency (lm, adj_index0);
 711           adj1 = ip_get_adjacency (lm, adj_index1);
 712
 713           if (PREDICT_FALSE (adj0->explicit_fib_index != ~0))
 714             {
 715               adj_index0 = ip6_fib_lookup_with_table
 716                 (im, adj0->explicit_fib_index, &ip0->dst_address);
 717               adj0 = ip_get_adjacency (lm, adj_index0);
 718             }
 719           if (PREDICT_FALSE (adj1->explicit_fib_index != ~0))
 720             {
 721               adj_index1 = ip6_fib_lookup_with_table
 722                 (im, adj1->explicit_fib_index, &ip1->dst_address);
 723               adj1 = ip_get_adjacency (lm, adj_index1);
 724             }
 725
 726           next0 = adj0->lookup_next_index;
 727           next1 = adj1->lookup_next_index;
 728
 729           /* Process hop-by-hop options if present */
 730           next0 = (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) ?
 731               IP_LOOKUP_NEXT_HOP_BY_HOP : next0;
 732           next1 = (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) ?
 733               IP_LOOKUP_NEXT_HOP_BY_HOP : next1;
 734
 735           vnet_buffer (p0)->ip.flow_hash =
 736             vnet_buffer(p1)->ip.flow_hash = 0;
 737
 738           if (PREDICT_FALSE(adj0->n_adj > 1))
 739             {
 740               flow_hash_config0 =
 741                 vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 742               vnet_buffer (p0)->ip.flow_hash =
 743                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 744             }
 745
 746           if (PREDICT_FALSE(adj1->n_adj > 1))
 747             {
 748               flow_hash_config1 =
 749                 vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 750
 751               vnet_buffer (p1)->ip.flow_hash =
 752                 ip6_compute_flow_hash (ip1, flow_hash_config1);
 753             }
 754
 755           ASSERT (adj0->n_adj > 0);
 756           ASSERT (adj1->n_adj > 0);
 757           ASSERT (is_pow2 (adj0->n_adj));
 758           ASSERT (is_pow2 (adj1->n_adj));
 759           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
 760           adj_index1 += (vnet_buffer (p1)->ip.flow_hash & (adj1->n_adj - 1));
 761
 762           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 763           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
 764
 765           vlib_increment_combined_counter
 766               (cm, cpu_index, adj_index0, 1,
 767                vlib_buffer_length_in_chain (vm, p0));
 768           vlib_increment_combined_counter
 769               (cm, cpu_index, adj_index1, 1,
 770                vlib_buffer_length_in_chain (vm, p1));
 771
 772           from += 2;
 773           to_next += 2;
 774           n_left_to_next -= 2;
 775           n_left_from -= 2;
 776
 777           wrong_next = (next0 != next) + 2*(next1 != next);
 778           if (PREDICT_FALSE (wrong_next != 0))
 779             {
 780               switch (wrong_next)
 781                 {
 782                 case 1:
 783                   /* A B A */
 784                   to_next[-2] = pi1;
 785                   to_next -= 1;
 786                   n_left_to_next += 1;
 787                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 788                   break;
 789
 790                 case 2:
 791                   /* A A B */
 792                   to_next -= 1;
 793                   n_left_to_next += 1;
 794                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 795                   break;
 796
 797                 case 3:
 798                   /* A B C */
 799                   to_next -= 2;
 800                   n_left_to_next += 2;
 801                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 802                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 803                   if (next0 == next1)
 804                     {
 805                       /* A B B */
 806                       vlib_put_next_frame (vm, node, next, n_left_to_next);
 807                       next = next1;
 808                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 809                     }
 810                 }
 811             }
 812         }
 813
 814       while (n_left_from > 0 && n_left_to_next > 0)
 815         {
 816           vlib_buffer_t * p0;
 817           ip6_header_t * ip0;
 818           u32 pi0, adj_index0;
 819           ip_lookup_next_t next0;
 820           ip_adjacency_t * adj0;
 821           u32 fib_index0, flow_hash_config0;
 822
 823           pi0 = from[0];
 824           to_next[0] = pi0;
 825
 826           p0 = vlib_get_buffer (vm, pi0);
 827
 828           ip0 = vlib_buffer_get_current (p0);
 829
 830           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 831           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 832             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 833
 834           flow_hash_config0 =
 835               vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 836
 837           adj_index0 = ip6_fib_lookup_with_table (im, fib_index0,
 838                                                   &ip0->dst_address);
 839
 840           adj0 = ip_get_adjacency (lm, adj_index0);
 841
 842           if (PREDICT_FALSE (adj0->explicit_fib_index != ~0))
 843             {
 844               adj_index0 = ip6_fib_lookup_with_table
 845                 (im, adj0->explicit_fib_index, &ip0->dst_address);
 846               adj0 = ip_get_adjacency (lm, adj_index0);
 847             }
 848
 849           next0 = adj0->lookup_next_index;
 850           next0 = (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) ?
 851               IP_LOOKUP_NEXT_HOP_BY_HOP : next0;
 852
 853           vnet_buffer (p0)->ip.flow_hash = 0;
 854
 855           if (PREDICT_FALSE(adj0->n_adj > 1))
 856             {
 857               flow_hash_config0 =
 858                 vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 859               vnet_buffer (p0)->ip.flow_hash =
 860                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 861             }
 862
 863           ASSERT (adj0->n_adj > 0);
 864           ASSERT (is_pow2 (adj0->n_adj));
 865           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
 866
 867           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 868
 869           vlib_increment_combined_counter
 870               (cm, cpu_index, adj_index0, 1,
 871                vlib_buffer_length_in_chain (vm, p0));
 872
 873           from += 1;
 874           to_next += 1;
 875           n_left_to_next -= 1;
 876           n_left_from -= 1;
 877
 878           if (PREDICT_FALSE (next0 != next))
 879             {
 880               n_left_to_next += 1;
 881               vlib_put_next_frame (vm, node, next, n_left_to_next);
 882               next = next0;
 883               vlib_get_next_frame (vm, node, next,
 884                                    to_next, n_left_to_next);
 885               to_next[0] = pi0;
 886               to_next += 1;
 887               n_left_to_next -= 1;
 888             }
 889         }
 890
 891       vlib_put_next_frame (vm, node, next, n_left_to_next);
 892     }
 893
 894   return frame->n_vectors;
 895 }
 896
 897 void ip6_adjacency_set_interface_route (vnet_main_t * vnm,
 898                                         ip_adjacency_t * adj,
 899                                         u32 sw_if_index,
 900                                         u32 if_address_index)
 901 {
 902   vnet_hw_interface_t * hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
 903   ip_lookup_next_t n;
 904   u32 node_index;
 905
 906   if (hw->hw_class_index == ethernet_hw_interface_class.index
 907       || hw->hw_class_index == srp_hw_interface_class.index)
 908     {
 909       n = IP_LOOKUP_NEXT_ARP;
 910       node_index = ip6_discover_neighbor_node.index;
 911       adj->if_address_index = if_address_index;
 912       adj->arp.next_hop.ip6.as_u64[0] = 0;
 913       adj->arp.next_hop.ip6.as_u64[1] = 0;
 914   }
 915   else
 916     {
 917       n = IP_LOOKUP_NEXT_REWRITE;
 918       node_index = ip6_rewrite_node.index;
 919     }
 920
 921  adj->lookup_next_index = n;
 922  adj->explicit_fib_index = ~0;
 923
 924  vnet_rewrite_for_sw_interface
 925    (vnm,
 926     VNET_L3_PACKET_TYPE_IP6,
 927     sw_if_index,
 928     node_index,
 929     VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST,
 930     &adj->rewrite_header,
 931     sizeof (adj->rewrite_data));
 932 }
 933
 934 static void
 935 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
 936                           ip6_main_t * im, u32 fib_index,
 937                           ip_interface_address_t * a)
 938 {
 939   ip_lookup_main_t * lm = &im->lookup_main;
 940   ip_adjacency_t * adj;
 941   ip6_address_t * address = ip_interface_address_get_address (lm, a);
 942   ip6_add_del_route_args_t x;
 943   vnet_hw_interface_t * hw_if = vnet_get_sup_hw_interface (vnm, sw_if_index);
 944   u32 classify_table_index;
 945
 946   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
 947   x.table_index_or_table_id = fib_index;
 948   x.flags = (IP6_ROUTE_FLAG_ADD
 949              | IP6_ROUTE_FLAG_FIB_INDEX
 950              | IP6_ROUTE_FLAG_NO_REDISTRIBUTE);
 951   x.dst_address = address[0];
 952   x.dst_address_length = a->address_length;
 953   x.n_add_adj = 0;
 954   x.add_adj = 0;
 955
 956   a->neighbor_probe_adj_index = ~0;
 957   if (a->address_length < 128)
 958     {
 959       adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 960                               &x.adj_index);
 961       ip6_adjacency_set_interface_route (vnm, adj, sw_if_index, a - lm->if_address_pool);
 962       ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
 963       ip6_add_del_route (im, &x);
 964       a->neighbor_probe_adj_index = x.adj_index;
 965     }
 966
 967   /* Add e.g. ::1/128 as local to this host. */
 968   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 969                           &x.adj_index);
 970
 971   classify_table_index = ~0;
 972   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 973     classify_table_index = lm->classify_table_index_by_sw_if_index [sw_if_index];
 974   if (classify_table_index != (u32) ~0)
 975     {
 976       adj->lookup_next_index = IP_LOOKUP_NEXT_CLASSIFY;
 977       adj->classify.table_index = classify_table_index;
 978     }
 979   else
 980     adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 981
 982   adj->if_address_index = a - lm->if_address_pool;
 983   adj->rewrite_header.sw_if_index = sw_if_index;
 984   adj->rewrite_header.max_l3_packet_bytes = hw_if->max_l3_packet_bytes[VLIB_RX];
 985   adj->rewrite_header.data_bytes = 0;
 986   ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
 987   x.dst_address_length = 128;
 988   ip6_add_del_route (im, &x);
 989 }
 990
 991 static void
 992 ip6_del_interface_routes (ip6_main_t * im, u32 fib_index,
 993                           ip6_address_t * address, u32 address_length)
 994 {
 995   ip6_add_del_route_args_t x;
 996
 997   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
 998   x.table_index_or_table_id = fib_index;
 999   x.flags = (IP6_ROUTE_FLAG_DEL
1000              | IP6_ROUTE_FLAG_FIB_INDEX
1001              | IP6_ROUTE_FLAG_NO_REDISTRIBUTE);
1002   x.dst_address = address[0];
1003   x.dst_address_length = address_length;
1004   x.adj_index = ~0;
1005   x.n_add_adj = 0;
1006   x.add_adj = 0;
1007
1008   if (address_length < 128)
1009     {
1010       /* Don't wipe out fe80::0/64 */
1011       if (address_length != 64 ||
1012           address[0].as_u64[0] != clib_net_to_host_u64(0xfe80000000000000ULL))
1013         ip6_add_del_route (im, &x);
1014     }
1015
1016   x.dst_address_length = 128;
1017   ip6_add_del_route (im, &x);
1018
1019   ip6_delete_matching_routes (im,
1020                               fib_index,
1021                               IP6_ROUTE_FLAG_FIB_INDEX,
1022                               address,
1023                               address_length);
1024 }
1025
1026 typedef struct {
1027     u32 sw_if_index;
1028     ip6_address_t address;
1029     u32 length;
1030 } ip6_interface_address_t;
1031
1032 static clib_error_t *
1033 ip6_add_del_interface_address_internal (vlib_main_t * vm,
1034                                         u32 sw_if_index,
1035                                         ip6_address_t * new_address,
1036                                         u32 new_length,
1037                                         u32 redistribute,
1038                                         u32 insert_routes,
1039                                         u32 is_del);
1040
1041 static clib_error_t *
1042 ip6_add_del_interface_address_internal (vlib_main_t * vm,
1043                                         u32 sw_if_index,
1044                                         ip6_address_t * address,
1045                                         u32 address_length,
1046                                         u32 redistribute,
1047                                         u32 insert_routes,
1048                                         u32 is_del)
1049 {
1050   vnet_main_t * vnm = vnet_get_main();
1051   ip6_main_t * im = &ip6_main;
1052   ip_lookup_main_t * lm = &im->lookup_main;
1053   clib_error_t * error;
1054   u32 if_address_index;
1055   ip6_address_fib_t ip6_af, * addr_fib = 0;
1056
1057   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1058   ip6_addr_fib_init (&ip6_af, address,
1059                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
1060   vec_add1 (addr_fib, ip6_af);
1061
1062   {
1063     uword elts_before = pool_elts (lm->if_address_pool);
1064
1065     error = ip_interface_address_add_del
1066       (lm,
1067        sw_if_index,
1068        addr_fib,
1069        address_length,
1070        is_del,
1071        &if_address_index);
1072     if (error)
1073       goto done;
1074
1075     /* Pool did not grow: add duplicate address. */
1076     if (elts_before == pool_elts (lm->if_address_pool))
1077       goto done;
1078   }
1079
1080   if (vnet_sw_interface_is_admin_up (vnm, sw_if_index) && insert_routes)
1081     {
1082       if (is_del)
1083         ip6_del_interface_routes (im, ip6_af.fib_index, address,
1084                                   address_length);
1085
1086       else
1087         ip6_add_interface_routes (vnm, sw_if_index,
1088                                   im, ip6_af.fib_index,
1089                                   pool_elt_at_index (lm->if_address_pool, if_address_index));
1090     }
1091
1092   {
1093     ip6_add_del_interface_address_callback_t * cb;
1094     vec_foreach (cb, im->add_del_interface_address_callbacks)
1095       cb->function (im, cb->function_opaque, sw_if_index,
1096                     address, address_length,
1097                     if_address_index,
1098                     is_del);
1099   }
1100
1101  done:
1102   vec_free (addr_fib);
1103   return error;
1104 }
1105
1106 clib_error_t *
1107 ip6_add_del_interface_address (vlib_main_t * vm, u32 sw_if_index,
1108                                ip6_address_t * address, u32 address_length,
1109                                u32 is_del)
1110 {
1111   return ip6_add_del_interface_address_internal
1112     (vm, sw_if_index, address, address_length,
1113      /* redistribute */ 1,
1114      /* insert_routes */ 1,
1115      is_del);
1116 }
1117
1118 clib_error_t *
1119 ip6_sw_interface_admin_up_down (vnet_main_t * vnm,
1120                                 u32 sw_if_index,
1121                                 u32 flags)
1122 {
1123   ip6_main_t * im = &ip6_main;
1124   ip_interface_address_t * ia;
1125   ip6_address_t * a;
1126   u32 is_admin_up, fib_index;
1127
1128   /* Fill in lookup tables with default table (0). */
1129   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1130
1131   vec_validate_init_empty (im->lookup_main.if_address_pool_index_by_sw_if_index, sw_if_index, ~0);
1132
1133   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1134
1135   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
1136
1137   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
1138                                 0 /* honor unnumbered */,
1139   ({
1140     a = ip_interface_address_get_address (&im->lookup_main, ia);
1141     if (is_admin_up)
1142       ip6_add_interface_routes (vnm, sw_if_index,
1143                                 im, fib_index,
1144                                 ia);
1145     else
1146       ip6_del_interface_routes (im, fib_index,
1147                                 a, ia->address_length);
1148   }));
1149
1150   return 0;
1151 }
1152
1153 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
1154
1155 clib_error_t *
1156 ip6_sw_interface_add_del (vnet_main_t * vnm,
1157                           u32 sw_if_index,
1158                           u32 is_add)
1159 {
1160   vlib_main_t * vm = vnm->vlib_main;
1161   ip6_main_t * im = &ip6_main;
1162   ip_lookup_main_t * lm = &im->lookup_main;
1163   u32 ci, cast;
1164
1165   for (cast = 0; cast < VNET_N_CAST; cast++)
1166     {
1167       ip_config_main_t * cm = &lm->rx_config_mains[cast];
1168       vnet_config_main_t * vcm = &cm->config_main;
1169
1170       /* FIXME multicast. */
1171       if (! vcm->node_index_by_feature_index)
1172         {
1173           char * start_nodes[] = { "ip6-input", };
1174           char * feature_nodes[] = {
1175             [IP6_RX_FEATURE_CHECK_ACCESS] = "ip6-inacl",
1176             [IP6_RX_FEATURE_IPSEC] = "ipsec-input-ip6",
1177             [IP6_RX_FEATURE_L2TPV3] = "l2tp-decap",
1178             [IP6_RX_FEATURE_VPATH]  = "vpath-input-ip6",
1179             [IP6_RX_FEATURE_LOOKUP] = "ip6-lookup",
1180           };
1181           vnet_config_init (vm, vcm,
1182                             start_nodes, ARRAY_LEN (start_nodes),
1183                             feature_nodes, ARRAY_LEN (feature_nodes));
1184         }
1185
1186       vec_validate_init_empty (cm->config_index_by_sw_if_index, sw_if_index, ~0);
1187       ci = cm->config_index_by_sw_if_index[sw_if_index];
1188
1189       if (is_add)
1190         ci = vnet_config_add_feature (vm, vcm,
1191                                       ci,
1192                                       IP6_RX_FEATURE_LOOKUP,
1193                                       /* config data */ 0,
1194                                       /* # bytes of config data */ 0);
1195       else
1196         ci = vnet_config_del_feature (vm, vcm,
1197                                       ci,
1198                                       IP6_RX_FEATURE_LOOKUP,
1199                                       /* config data */ 0,
1200                                       /* # bytes of config data */ 0);
1201
1202       cm->config_index_by_sw_if_index[sw_if_index] = ci;
1203     }
1204   return /* no error */ 0;
1205 }
1206
1207 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
1208
1209 VLIB_REGISTER_NODE (ip6_lookup_node) = {
1210   .function = ip6_lookup,
1211   .name = "ip6-lookup",
1212   .vector_size = sizeof (u32),
1213
1214   .n_next_nodes = IP_LOOKUP_N_NEXT,
1215   .next_nodes = IP6_LOOKUP_NEXT_NODES,
1216 };
1217
1218 typedef struct {
1219   /* Adjacency taken. */
1220   u32 adj_index;
1221   u32 flow_hash;
1222   u32 fib_index;
1223
1224   /* Packet data, possibly *after* rewrite. */
1225   u8 packet_data[64 - 1*sizeof(u32)];
1226 } ip6_forward_next_trace_t;
1227
1228 static u8 * format_ip6_forward_next_trace (u8 * s, va_list * args)
1229 {
1230   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1231   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1232   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
1233   vnet_main_t * vnm = vnet_get_main();
1234   ip6_main_t * im = &ip6_main;
1235   ip_adjacency_t * adj;
1236   uword indent = format_get_indent (s);
1237
1238   adj = ip_get_adjacency (&im->lookup_main, t->adj_index);
1239   s = format (s, "fib %d adj-idx %d : %U flow hash: 0x%08x",
1240               t->fib_index, t->adj_index, format_ip_adjacency,
1241               vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1242   switch (adj->lookup_next_index)
1243     {
1244     case IP_LOOKUP_NEXT_REWRITE:
1245       s = format (s, "\n%U%U",
1246                   format_white_space, indent,
1247                   format_ip_adjacency_packet_data,
1248                   vnm, &im->lookup_main, t->adj_index,
1249                   t->packet_data, sizeof (t->packet_data));
1250       break;
1251
1252     default:
1253       break;
1254     }
1255
1256   return s;
1257 }
1258
1259 /* Common trace function for all ip6-forward next nodes. */
1260 void
1261 ip6_forward_next_trace (vlib_main_t * vm,
1262                         vlib_node_runtime_t * node,
1263                         vlib_frame_t * frame,
1264                         vlib_rx_or_tx_t which_adj_index)
1265 {
1266   u32 * from, n_left;
1267   ip6_main_t * im = &ip6_main;
1268
1269   n_left = frame->n_vectors;
1270   from = vlib_frame_vector_args (frame);
1271
1272   while (n_left >= 4)
1273     {
1274       u32 bi0, bi1;
1275       vlib_buffer_t * b0, * b1;
1276       ip6_forward_next_trace_t * t0, * t1;
1277
1278       /* Prefetch next iteration. */
1279       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1280       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1281
1282       bi0 = from[0];
1283       bi1 = from[1];
1284
1285       b0 = vlib_get_buffer (vm, bi0);
1286       b1 = vlib_get_buffer (vm, bi1);
1287
1288       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1289         {
1290           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1291           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1292           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1293           t0->fib_index = vec_elt (im->fib_index_by_sw_if_index,
1294                              vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1295           clib_memcpy (t0->packet_data,
1296                   vlib_buffer_get_current (b0),
1297                   sizeof (t0->packet_data));
1298         }
1299       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1300         {
1301           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1302           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1303           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1304           t1->fib_index = vec_elt (im->fib_index_by_sw_if_index,
1305                              vnet_buffer(b1)->sw_if_index[VLIB_RX]);
1306           clib_memcpy (t1->packet_data,
1307                   vlib_buffer_get_current (b1),
1308                   sizeof (t1->packet_data));
1309         }
1310       from += 2;
1311       n_left -= 2;
1312     }
1313
1314   while (n_left >= 1)
1315     {
1316       u32 bi0;
1317       vlib_buffer_t * b0;
1318       ip6_forward_next_trace_t * t0;
1319
1320       bi0 = from[0];
1321
1322       b0 = vlib_get_buffer (vm, bi0);
1323
1324       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1325         {
1326           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1327           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1328           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1329           t0->fib_index = vec_elt (im->fib_index_by_sw_if_index,
1330                              vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1331           clib_memcpy (t0->packet_data,
1332                   vlib_buffer_get_current (b0),
1333                   sizeof (t0->packet_data));
1334         }
1335       from += 1;
1336       n_left -= 1;
1337     }
1338 }
1339
1340 static uword
1341 ip6_drop_or_punt (vlib_main_t * vm,
1342                   vlib_node_runtime_t * node,
1343                   vlib_frame_t * frame,
1344                   ip6_error_t error_code)
1345 {
1346   u32 * buffers = vlib_frame_vector_args (frame);
1347   uword n_packets = frame->n_vectors;
1348
1349   vlib_error_drop_buffers (vm, node,
1350                            buffers,
1351                            /* stride */ 1,
1352                            n_packets,
1353                            /* next */ 0,
1354                            ip6_input_node.index,
1355                            error_code);
1356
1357   if (node->flags & VLIB_NODE_FLAG_TRACE)
1358     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1359
1360   return n_packets;
1361 }
1362
1363 static uword
1364 ip6_drop (vlib_main_t * vm,
1365           vlib_node_runtime_t * node,
1366           vlib_frame_t * frame)
1367 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_DROP); }
1368
1369 static uword
1370 ip6_punt (vlib_main_t * vm,
1371           vlib_node_runtime_t * node,
1372           vlib_frame_t * frame)
1373 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_PUNT); }
1374
1375 static uword
1376 ip6_miss (vlib_main_t * vm,
1377           vlib_node_runtime_t * node,
1378           vlib_frame_t * frame)
1379 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_DST_LOOKUP_MISS); }
1380
1381 VLIB_REGISTER_NODE (ip6_drop_node,static) = {
1382   .function = ip6_drop,
1383   .name = "ip6-drop",
1384   .vector_size = sizeof (u32),
1385
1386   .format_trace = format_ip6_forward_next_trace,
1387
1388   .n_next_nodes = 1,
1389   .next_nodes = {
1390     [0] = "error-drop",
1391   },
1392 };
1393
1394 VLIB_REGISTER_NODE (ip6_punt_node,static) = {
1395   .function = ip6_punt,
1396   .name = "ip6-punt",
1397   .vector_size = sizeof (u32),
1398
1399   .format_trace = format_ip6_forward_next_trace,
1400
1401   .n_next_nodes = 1,
1402   .next_nodes = {
1403     [0] = "error-punt",
1404   },
1405 };
1406
1407 VLIB_REGISTER_NODE (ip6_miss_node,static) = {
1408   .function = ip6_miss,
1409   .name = "ip6-miss",
1410   .vector_size = sizeof (u32),
1411
1412   .format_trace = format_ip6_forward_next_trace,
1413
1414   .n_next_nodes = 1,
1415   .next_nodes = {
1416     [0] = "error-drop",
1417   },
1418 };
1419
1420 VLIB_REGISTER_NODE (ip6_multicast_node,static) = {
1421   .function = ip6_drop,
1422   .name = "ip6-multicast",
1423   .vector_size = sizeof (u32),
1424
1425   .format_trace = format_ip6_forward_next_trace,
1426
1427   .n_next_nodes = 1,
1428   .next_nodes = {
1429     [0] = "error-drop",
1430   },
1431 };
1432
1433 /* Compute TCP/UDP/ICMP6 checksum in software. */
1434 u16 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, ip6_header_t * ip0, int *bogus_lengthp)
1435 {
1436   ip_csum_t sum0;
1437   u16 sum16, payload_length_host_byte_order;
1438   u32 i, n_this_buffer, n_bytes_left;
1439   u32 headers_size = sizeof(ip0[0]);
1440   void * data_this_buffer;
1441
1442   ASSERT(bogus_lengthp);
1443   *bogus_lengthp = 0;
1444
1445   /* Initialize checksum with ip header. */
1446   sum0 = ip0->payload_length + clib_host_to_net_u16 (ip0->protocol);
1447   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1448   data_this_buffer = (void *) (ip0 + 1);
1449
1450   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
1451     {
1452       sum0 = ip_csum_with_carry (sum0,
1453                                  clib_mem_unaligned (&ip0->src_address.as_uword[i], uword));
1454       sum0 = ip_csum_with_carry (sum0,
1455                                  clib_mem_unaligned (&ip0->dst_address.as_uword[i], uword));
1456     }
1457
1458   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1459   if (PREDICT_FALSE (ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1460     {
1461       u32  skip_bytes;
1462       ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t  *)data_this_buffer;
1463
1464       /* validate really icmp6 next */
1465       ASSERT(ext_hdr->next_hdr == IP_PROTOCOL_ICMP6);
1466
1467       skip_bytes = 8* (1 + ext_hdr->n_data_u64s);
1468       data_this_buffer  = (void *)((u8 *)data_this_buffer + skip_bytes);
1469
1470       payload_length_host_byte_order  -= skip_bytes;
1471       headers_size += skip_bytes;
1472    }
1473
1474   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1475 #if DPDK > 0
1476   if (p0)
1477   {
1478     struct rte_mbuf *mb = rte_mbuf_from_vlib_buffer(p0);
1479     u8 nb_segs = mb->nb_segs;
1480
1481     n_this_buffer = (p0->current_length > headers_size ?
1482                      p0->current_length - headers_size : 0);
1483     while (n_bytes_left)
1484       {
1485         sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1486         n_bytes_left -= n_this_buffer;
1487
1488         mb = mb->next;
1489         nb_segs--;
1490         if ((nb_segs == 0) || (mb == 0))
1491           break;
1492
1493         data_this_buffer = rte_ctrlmbuf_data(mb);
1494         n_this_buffer = mb->data_len;
1495       }
1496     if (n_bytes_left || nb_segs)
1497       {
1498         *bogus_lengthp = 1;
1499         return 0xfefe;
1500       }
1501   }
1502   else sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1503 #else
1504   if (p0 && n_this_buffer + headers_size  > p0->current_length)
1505     n_this_buffer = p0->current_length > headers_size  ? p0->current_length - headers_size  : 0;
1506   while (1)
1507     {
1508       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1509       n_bytes_left -= n_this_buffer;
1510       if (n_bytes_left == 0)
1511         break;
1512
1513       if (!(p0->flags & VLIB_BUFFER_NEXT_PRESENT))
1514         {
1515           *bogus_lengthp = 1;
1516           return 0xfefe;
1517         }
1518       p0 = vlib_get_buffer (vm, p0->next_buffer);
1519       data_this_buffer = vlib_buffer_get_current (p0);
1520       n_this_buffer = p0->current_length;
1521     }
1522 #endif /* DPDK */
1523
1524   sum16 = ~ ip_csum_fold (sum0);
1525
1526   return sum16;
1527 }
1528
1529 u32 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1530 {
1531   ip6_header_t * ip0 = vlib_buffer_get_current (p0);
1532   udp_header_t * udp0;
1533   u16 sum16;
1534   int bogus_length;
1535
1536   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1537   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1538           || ip0->protocol == IP_PROTOCOL_ICMP6
1539           || ip0->protocol == IP_PROTOCOL_UDP
1540           || ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1541
1542   udp0 = (void *) (ip0 + 1);
1543   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1544     {
1545       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1546                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1547       return p0->flags;
1548     }
1549
1550   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1551
1552   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1553                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1554
1555   return p0->flags;
1556 }
1557
1558 static uword
1559 ip6_local (vlib_main_t * vm,
1560            vlib_node_runtime_t * node,
1561            vlib_frame_t * frame)
1562 {
1563   ip6_main_t * im = &ip6_main;
1564   ip_lookup_main_t * lm = &im->lookup_main;
1565   ip_local_next_t next_index;
1566   u32 * from, * to_next, n_left_from, n_left_to_next;
1567   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
1568
1569   from = vlib_frame_vector_args (frame);
1570   n_left_from = frame->n_vectors;
1571   next_index = node->cached_next_index;
1572
1573   if (node->flags & VLIB_NODE_FLAG_TRACE)
1574     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1575
1576   while (n_left_from > 0)
1577     {
1578       vlib_get_next_frame (vm, node, next_index,
1579                            to_next, n_left_to_next);
1580
1581       while (n_left_from >= 4 && n_left_to_next >= 2)
1582         {
1583           vlib_buffer_t * p0, * p1;
1584           ip6_header_t * ip0, * ip1;
1585           udp_header_t * udp0, * udp1;
1586           u32 pi0, ip_len0, udp_len0, flags0, next0;
1587           u32 pi1, ip_len1, udp_len1, flags1, next1;
1588           i32 len_diff0, len_diff1;
1589           u8 error0, type0, good_l4_checksum0;
1590           u8 error1, type1, good_l4_checksum1;
1591
1592           pi0 = to_next[0] = from[0];
1593           pi1 = to_next[1] = from[1];
1594           from += 2;
1595           n_left_from -= 2;
1596           to_next += 2;
1597           n_left_to_next -= 2;
1598
1599           p0 = vlib_get_buffer (vm, pi0);
1600           p1 = vlib_get_buffer (vm, pi1);
1601
1602           ip0 = vlib_buffer_get_current (p0);
1603           ip1 = vlib_buffer_get_current (p1);
1604
1605           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1606           type1 = lm->builtin_protocol_by_ip_protocol[ip1->protocol];
1607
1608           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1609           next1 = lm->local_next_by_ip_protocol[ip1->protocol];
1610
1611           flags0 = p0->flags;
1612           flags1 = p1->flags;
1613
1614           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1615           good_l4_checksum1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1616
1617           udp0 = ip6_next_header (ip0);
1618           udp1 = ip6_next_header (ip1);
1619
1620           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1621           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1622           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UDP && udp1->checksum == 0;
1623
1624           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1625           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1626
1627           /* Verify UDP length. */
1628           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1629           ip_len1 = clib_net_to_host_u16 (ip1->payload_length);
1630           udp_len0 = clib_net_to_host_u16 (udp0->length);
1631           udp_len1 = clib_net_to_host_u16 (udp1->length);
1632
1633           len_diff0 = ip_len0 - udp_len0;
1634           len_diff1 = ip_len1 - udp_len1;
1635
1636           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1637           len_diff1 = type1 == IP_BUILTIN_PROTOCOL_UDP ? len_diff1 : 0;
1638
1639           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1640                              && ! good_l4_checksum0
1641                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1642             {
1643               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1644               good_l4_checksum0 =
1645                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1646             }
1647           if (PREDICT_FALSE (type1 != IP_BUILTIN_PROTOCOL_UNKNOWN
1648                              && ! good_l4_checksum1
1649                              && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1650             {
1651               flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, p1);
1652               good_l4_checksum1 =
1653                 (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1654             }
1655
1656           error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL;
1657
1658           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1659           error1 = len_diff1 < 0 ? IP6_ERROR_UDP_LENGTH : error1;
1660
1661           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1662           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1663           error0 = (! good_l4_checksum0
1664                     ? IP6_ERROR_UDP_CHECKSUM + type0
1665                     : error0);
1666           error1 = (! good_l4_checksum1
1667                     ? IP6_ERROR_UDP_CHECKSUM + type1
1668                     : error1);
1669
1670           /* Drop packets from unroutable hosts. */
1671           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1672           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL && type0 != IP_BUILTIN_PROTOCOL_ICMP)
1673             {
1674               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1675               error0 = (lm->miss_adj_index == src_adj_index0
1676                         ? IP6_ERROR_SRC_LOOKUP_MISS
1677                         : error0);
1678             }
1679           if (error1 == IP6_ERROR_UNKNOWN_PROTOCOL && type1 != IP_BUILTIN_PROTOCOL_ICMP)
1680             {
1681               u32 src_adj_index1 = ip6_src_lookup_for_packet (im, p1, ip1);
1682               error1 = (lm->miss_adj_index == src_adj_index1
1683                         ? IP6_ERROR_SRC_LOOKUP_MISS
1684                         : error1);
1685             }
1686
1687           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1688           next1 = error1 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1689
1690           p0->error = error_node->errors[error0];
1691           p1->error = error_node->errors[error1];
1692
1693           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1694                                            to_next, n_left_to_next,
1695                                            pi0, pi1, next0, next1);
1696         }
1697
1698       while (n_left_from > 0 && n_left_to_next > 0)
1699         {
1700           vlib_buffer_t * p0;
1701           ip6_header_t * ip0;
1702           udp_header_t * udp0;
1703           u32 pi0, ip_len0, udp_len0, flags0, next0;
1704           i32 len_diff0;
1705           u8 error0, type0, good_l4_checksum0;
1706
1707           pi0 = to_next[0] = from[0];
1708           from += 1;
1709           n_left_from -= 1;
1710           to_next += 1;
1711           n_left_to_next -= 1;
1712
1713           p0 = vlib_get_buffer (vm, pi0);
1714
1715           ip0 = vlib_buffer_get_current (p0);
1716
1717           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1718           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1719
1720           flags0 = p0->flags;
1721
1722           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1723
1724           udp0 = ip6_next_header (ip0);
1725
1726           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1727           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1728
1729           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1730
1731           /* Verify UDP length. */
1732           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1733           udp_len0 = clib_net_to_host_u16 (udp0->length);
1734
1735           len_diff0 = ip_len0 - udp_len0;
1736
1737           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1738
1739           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1740                              && ! good_l4_checksum0
1741                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1742             {
1743               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1744               good_l4_checksum0 =
1745                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1746             }
1747
1748           error0 = IP6_ERROR_UNKNOWN_PROTOCOL;
1749
1750           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1751
1752           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1753           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1754           error0 = (! good_l4_checksum0
1755                     ? IP6_ERROR_UDP_CHECKSUM + type0
1756                     : error0);
1757
1758           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1759           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL && type0 != IP_BUILTIN_PROTOCOL_ICMP)
1760             {
1761               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1762               error0 = (lm->miss_adj_index == src_adj_index0
1763                         ? IP6_ERROR_SRC_LOOKUP_MISS
1764                         : error0);
1765             }
1766
1767           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1768
1769           p0->error = error_node->errors[error0];
1770
1771           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1772                                            to_next, n_left_to_next,
1773                                            pi0, next0);
1774         }
1775
1776       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1777     }
1778
1779   return frame->n_vectors;
1780 }
1781
1782 VLIB_REGISTER_NODE (ip6_local_node,static) = {
1783   .function = ip6_local,
1784   .name = "ip6-local",
1785   .vector_size = sizeof (u32),
1786
1787   .format_trace = format_ip6_forward_next_trace,
1788
1789   .n_next_nodes = IP_LOCAL_N_NEXT,
1790   .next_nodes = {
1791     [IP_LOCAL_NEXT_DROP] = "error-drop",
1792     [IP_LOCAL_NEXT_PUNT] = "error-punt",
1793     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1794     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1795   },
1796 };
1797
1798 void ip6_register_protocol (u32 protocol, u32 node_index)
1799 {
1800   vlib_main_t * vm = vlib_get_main();
1801   ip6_main_t * im = &ip6_main;
1802   ip_lookup_main_t * lm = &im->lookup_main;
1803
1804   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1805   lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip6_local_node.index, node_index);
1806 }
1807
1808 typedef enum {
1809   IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1810   IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX,
1811   IP6_DISCOVER_NEIGHBOR_N_NEXT,
1812 } ip6_discover_neighbor_next_t;
1813
1814 typedef enum {
1815   IP6_DISCOVER_NEIGHBOR_ERROR_DROP,
1816   IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT,
1817 } ip6_discover_neighbor_error_t;
1818
1819 static uword
1820 ip6_discover_neighbor (vlib_main_t * vm,
1821                        vlib_node_runtime_t * node,
1822                        vlib_frame_t * frame)
1823 {
1824   vnet_main_t * vnm = vnet_get_main();
1825   ip6_main_t * im = &ip6_main;
1826   ip_lookup_main_t * lm = &im->lookup_main;
1827   u32 * from, * to_next_drop;
1828   uword n_left_from, n_left_to_next_drop;
1829   static f64 time_last_seed_change = -1e100;
1830   static u32 hash_seeds[3];
1831   static uword hash_bitmap[256 / BITS (uword)];
1832   f64 time_now;
1833   int bogus_length;
1834
1835   if (node->flags & VLIB_NODE_FLAG_TRACE)
1836     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1837
1838   time_now = vlib_time_now (vm);
1839   if (time_now - time_last_seed_change > 1e-3)
1840     {
1841       uword i;
1842       u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
1843                                              sizeof (hash_seeds));
1844       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1845         hash_seeds[i] = r[i];
1846
1847       /* Mark all hash keys as been not-seen before. */
1848       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1849         hash_bitmap[i] = 0;
1850
1851       time_last_seed_change = time_now;
1852     }
1853
1854   from = vlib_frame_vector_args (frame);
1855   n_left_from = frame->n_vectors;
1856
1857   while (n_left_from > 0)
1858     {
1859       vlib_get_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1860                            to_next_drop, n_left_to_next_drop);
1861
1862       while (n_left_from > 0 && n_left_to_next_drop > 0)
1863         {
1864           vlib_buffer_t * p0;
1865           ip6_header_t * ip0;
1866           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
1867           uword bm0;
1868           ip_adjacency_t * adj0;
1869           vnet_hw_interface_t * hw_if0;
1870           u32 next0;
1871
1872           pi0 = from[0];
1873
1874           p0 = vlib_get_buffer (vm, pi0);
1875
1876           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1877
1878           ip0 = vlib_buffer_get_current (p0);
1879
1880           adj0 = ip_get_adjacency (lm, adj_index0);
1881
1882           if (adj0->arp.next_hop.ip6.as_u64[0] ||
1883               adj0->arp.next_hop.ip6.as_u64[1]) {
1884             ip0->dst_address.as_u64[0] = adj0->arp.next_hop.ip6.as_u64[0];
1885             ip0->dst_address.as_u64[1] = adj0->arp.next_hop.ip6.as_u64[1];
1886           }
1887
1888           a0 = hash_seeds[0];
1889           b0 = hash_seeds[1];
1890           c0 = hash_seeds[2];
1891
1892           sw_if_index0 = adj0->rewrite_header.sw_if_index;
1893           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
1894
1895           a0 ^= sw_if_index0;
1896           b0 ^= ip0->dst_address.as_u32[0];
1897           c0 ^= ip0->dst_address.as_u32[1];
1898
1899           hash_v3_mix32 (a0, b0, c0);
1900
1901           b0 ^= ip0->dst_address.as_u32[2];
1902           c0 ^= ip0->dst_address.as_u32[3];
1903
1904           hash_v3_finalize32 (a0, b0, c0);
1905
1906           c0 &= BITS (hash_bitmap) - 1;
1907           c0 = c0 / BITS (uword);
1908           m0 = (uword) 1 << (c0 % BITS (uword));
1909
1910           bm0 = hash_bitmap[c0];
1911           drop0 = (bm0 & m0) != 0;
1912
1913           /* Mark it as seen. */
1914           hash_bitmap[c0] = bm0 | m0;
1915
1916           from += 1;
1917           n_left_from -= 1;
1918           to_next_drop[0] = pi0;
1919           to_next_drop += 1;
1920           n_left_to_next_drop -= 1;
1921
1922           hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1923
1924           /* If the interface is link-down, drop the pkt */
1925           if (!(hw_if0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))
1926             drop0 = 1;
1927
1928           p0->error =
1929             node->errors[drop0 ? IP6_DISCOVER_NEIGHBOR_ERROR_DROP
1930                          : IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT];
1931           if (drop0)
1932             continue;
1933
1934           {
1935             u32 bi0 = 0;
1936             icmp6_neighbor_solicitation_header_t * h0;
1937             vlib_buffer_t * b0;
1938
1939             h0 = vlib_packet_template_get_packet
1940               (vm, &im->discover_neighbor_packet_template, &bi0);
1941
1942             /*
1943              * Build ethernet header.
1944              * Choose source address based on destination lookup
1945              * adjacency.
1946              */
1947             ip6_src_address_for_packet (im, p0, &h0->ip.src_address,
1948                                         sw_if_index0);
1949
1950             /*
1951              * Destination address is a solicited node multicast address.
1952              * We need to fill in
1953              * the low 24 bits with low 24 bits of target's address.
1954              */
1955             h0->ip.dst_address.as_u8[13] = ip0->dst_address.as_u8[13];
1956             h0->ip.dst_address.as_u8[14] = ip0->dst_address.as_u8[14];
1957             h0->ip.dst_address.as_u8[15] = ip0->dst_address.as_u8[15];
1958
1959             h0->neighbor.target_address = ip0->dst_address;
1960
1961             clib_memcpy (h0->link_layer_option.ethernet_address,
1962                     hw_if0->hw_address, vec_len (hw_if0->hw_address));
1963
1964             /* $$$$ appears we need this; why is the checksum non-zero? */
1965             h0->neighbor.icmp.checksum = 0;
1966             h0->neighbor.icmp.checksum =
1967               ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h0->ip,
1968                                                  &bogus_length);
1969
1970             ASSERT (bogus_length == 0);
1971
1972             vlib_buffer_copy_trace_flag (vm, p0, bi0);
1973             b0 = vlib_get_buffer (vm, bi0);
1974             vnet_buffer (b0)->sw_if_index[VLIB_TX]
1975               = vnet_buffer (p0)->sw_if_index[VLIB_TX];
1976
1977             /* Add rewrite/encap string. */
1978             vnet_rewrite_one_header (adj0[0], h0,
1979                                      sizeof (ethernet_header_t));
1980             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
1981
1982             next0 = IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX;
1983
1984             vlib_set_next_frame_buffer (vm, node, next0, bi0);
1985           }
1986         }
1987
1988       vlib_put_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1989                            n_left_to_next_drop);
1990     }
1991
1992   return frame->n_vectors;
1993 }
1994
1995 static char * ip6_discover_neighbor_error_strings[] = {
1996   [IP6_DISCOVER_NEIGHBOR_ERROR_DROP] = "address overflow drops",
1997   [IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT]
1998   = "neighbor solicitations sent",
1999 };
2000
2001 VLIB_REGISTER_NODE (ip6_discover_neighbor_node) = {
2002   .function = ip6_discover_neighbor,
2003   .name = "ip6-discover-neighbor",
2004   .vector_size = sizeof (u32),
2005
2006   .format_trace = format_ip6_forward_next_trace,
2007
2008   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
2009   .error_strings = ip6_discover_neighbor_error_strings,
2010
2011   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
2012   .next_nodes = {
2013     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop",
2014     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
2015   },
2016 };
2017
2018 clib_error_t *
2019 ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index)
2020 {
2021   vnet_main_t * vnm = vnet_get_main();
2022   ip6_main_t * im = &ip6_main;
2023   icmp6_neighbor_solicitation_header_t * h;
2024   ip6_address_t * src;
2025   ip_interface_address_t * ia;
2026   ip_adjacency_t * adj;
2027   vnet_hw_interface_t * hi;
2028   vnet_sw_interface_t * si;
2029   vlib_buffer_t * b;
2030   u32 bi = 0;
2031   int bogus_length;
2032
2033   si = vnet_get_sw_interface (vnm, sw_if_index);
2034
2035   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2036     {
2037       return clib_error_return (0, "%U: interface %U down",
2038                                 format_ip6_address, dst,
2039                                 format_vnet_sw_if_index_name, vnm,
2040                                 sw_if_index);
2041     }
2042
2043   src = ip6_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2044   if (! src)
2045     {
2046       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2047       return clib_error_return
2048         (0, "no matching interface address for destination %U (interface %U)",
2049          format_ip6_address, dst,
2050          format_vnet_sw_if_index_name, vnm, sw_if_index);
2051     }
2052
2053   h = vlib_packet_template_get_packet (vm, &im->discover_neighbor_packet_template, &bi);
2054
2055   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2056
2057   /* Destination address is a solicited node multicast address.  We need to fill in
2058      the low 24 bits with low 24 bits of target's address. */
2059   h->ip.dst_address.as_u8[13] = dst->as_u8[13];
2060   h->ip.dst_address.as_u8[14] = dst->as_u8[14];
2061   h->ip.dst_address.as_u8[15] = dst->as_u8[15];
2062
2063   h->ip.src_address = src[0];
2064   h->neighbor.target_address = dst[0];
2065
2066   clib_memcpy (h->link_layer_option.ethernet_address, hi->hw_address, vec_len (hi->hw_address));
2067
2068   h->neighbor.icmp.checksum =
2069     ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length);
2070   ASSERT(bogus_length == 0);
2071
2072   b = vlib_get_buffer (vm, bi);
2073   vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2074
2075   /* Add encapsulation string for software interface (e.g. ethernet header). */
2076   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
2077   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2078   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2079
2080   {
2081     vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
2082     u32 * to_next = vlib_frame_vector_args (f);
2083     to_next[0] = bi;
2084     f->n_vectors = 1;
2085     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2086   }
2087
2088   return /* no error */ 0;
2089 }
2090
2091 typedef enum {
2092   IP6_REWRITE_NEXT_DROP,
2093 } ip6_rewrite_next_t;
2094
2095 always_inline uword
2096 ip6_rewrite_inline (vlib_main_t * vm,
2097                     vlib_node_runtime_t * node,
2098                     vlib_frame_t * frame,
2099                     int rewrite_for_locally_received_packets)
2100 {
2101   ip_lookup_main_t * lm = &ip6_main.lookup_main;
2102   u32 * from = vlib_frame_vector_args (frame);
2103   u32 n_left_from, n_left_to_next, * to_next, next_index;
2104   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
2105   vlib_rx_or_tx_t adj_rx_tx = rewrite_for_locally_received_packets ? VLIB_RX : VLIB_TX;
2106
2107   n_left_from = frame->n_vectors;
2108   next_index = node->cached_next_index;
2109   u32 cpu_index = os_get_cpu_number();
2110
2111   while (n_left_from > 0)
2112     {
2113       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2114
2115       while (n_left_from >= 4 && n_left_to_next >= 2)
2116         {
2117           ip_adjacency_t * adj0, * adj1;
2118           vlib_buffer_t * p0, * p1;
2119           ip6_header_t * ip0, * ip1;
2120           u32 pi0, rw_len0, next0, error0, adj_index0;
2121           u32 pi1, rw_len1, next1, error1, adj_index1;
2122
2123           /* Prefetch next iteration. */
2124           {
2125             vlib_buffer_t * p2, * p3;
2126
2127             p2 = vlib_get_buffer (vm, from[2]);
2128             p3 = vlib_get_buffer (vm, from[3]);
2129
2130             vlib_prefetch_buffer_header (p2, LOAD);
2131             vlib_prefetch_buffer_header (p3, LOAD);
2132
2133             CLIB_PREFETCH (p2->pre_data, 32, STORE);
2134             CLIB_PREFETCH (p3->pre_data, 32, STORE);
2135
2136             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2137             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2138           }
2139
2140           pi0 = to_next[0] = from[0];
2141           pi1 = to_next[1] = from[1];
2142
2143           from += 2;
2144           n_left_from -= 2;
2145           to_next += 2;
2146           n_left_to_next -= 2;
2147
2148           p0 = vlib_get_buffer (vm, pi0);
2149           p1 = vlib_get_buffer (vm, pi1);
2150
2151           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2152           adj_index1 = vnet_buffer (p1)->ip.adj_index[adj_rx_tx];
2153
2154           /* We should never rewrite a pkt using the MISS adjacency */
2155           ASSERT(adj_index0 && adj_index1);
2156
2157           ip0 = vlib_buffer_get_current (p0);
2158           ip1 = vlib_buffer_get_current (p1);
2159
2160           error0 = error1 = IP6_ERROR_NONE;
2161
2162           if (! rewrite_for_locally_received_packets)
2163             {
2164               i32 hop_limit0 = ip0->hop_limit, hop_limit1 = ip1->hop_limit;
2165
2166               /* Input node should have reject packets with hop limit 0. */
2167               ASSERT (ip0->hop_limit > 0);
2168               ASSERT (ip1->hop_limit > 0);
2169
2170               hop_limit0 -= 1;
2171               hop_limit1 -= 1;
2172
2173               ip0->hop_limit = hop_limit0;
2174               ip1->hop_limit = hop_limit1;
2175
2176               error0 = hop_limit0 <= 0 ? IP6_ERROR_TIME_EXPIRED : error0;
2177               error1 = hop_limit1 <= 0 ? IP6_ERROR_TIME_EXPIRED : error1;
2178             }
2179
2180           adj0 = ip_get_adjacency (lm, adj_index0);
2181           adj1 = ip_get_adjacency (lm, adj_index1);
2182
2183           if (rewrite_for_locally_received_packets)
2184             {
2185               /*
2186                * If someone sends e.g. an icmp6 w/ src = dst = interface addr,
2187                * we end up here with a local adjacency in hand
2188                */
2189               if (PREDICT_FALSE(adj0->lookup_next_index
2190                                 == IP_LOOKUP_NEXT_LOCAL))
2191                 error0 = IP6_ERROR_SPOOFED_LOCAL_PACKETS;
2192               if (PREDICT_FALSE(adj1->lookup_next_index
2193                                 == IP_LOOKUP_NEXT_LOCAL))
2194                 error1 = IP6_ERROR_SPOOFED_LOCAL_PACKETS;
2195             }
2196
2197           rw_len0 = adj0[0].rewrite_header.data_bytes;
2198           rw_len1 = adj1[0].rewrite_header.data_bytes;
2199
2200           vlib_increment_combined_counter (&lm->adjacency_counters,
2201                                            cpu_index,
2202                                            adj_index0,
2203                                            /* packet increment */ 0,
2204                                            /* byte increment */ rw_len0);
2205           vlib_increment_combined_counter (&lm->adjacency_counters,
2206                                            cpu_index,
2207                                            adj_index1,
2208                                            /* packet increment */ 0,
2209                                            /* byte increment */ rw_len1);
2210
2211           /* Check MTU of outgoing interface. */
2212           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2213                     ? IP6_ERROR_MTU_EXCEEDED
2214                     : error0);
2215           error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
2216                     ? IP6_ERROR_MTU_EXCEEDED
2217                     : error1);
2218
2219           p0->current_data -= rw_len0;
2220           p1->current_data -= rw_len1;
2221
2222           p0->current_length += rw_len0;
2223           p1->current_length += rw_len1;
2224
2225           vnet_buffer (p0)->sw_if_index[VLIB_TX] = adj0[0].rewrite_header.sw_if_index;
2226           vnet_buffer (p1)->sw_if_index[VLIB_TX] = adj1[0].rewrite_header.sw_if_index;
2227
2228           next0 = (error0 == IP6_ERROR_NONE) ?
2229             adj0[0].rewrite_header.next_index : IP6_REWRITE_NEXT_DROP;
2230           next1 = (error1 == IP6_ERROR_NONE) ?
2231             adj1[0].rewrite_header.next_index : IP6_REWRITE_NEXT_DROP;
2232
2233           /* Guess we are only writing on simple Ethernet header. */
2234           vnet_rewrite_two_headers (adj0[0], adj1[0],
2235                                     ip0, ip1,
2236                                     sizeof (ethernet_header_t));
2237
2238           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2239                                            to_next, n_left_to_next,
2240                                            pi0, pi1, next0, next1);
2241         }
2242
2243       while (n_left_from > 0 && n_left_to_next > 0)
2244         {
2245           ip_adjacency_t * adj0;
2246           vlib_buffer_t * p0;
2247           ip6_header_t * ip0;
2248           u32 pi0, rw_len0;
2249           u32 adj_index0, next0, error0;
2250
2251           pi0 = to_next[0] = from[0];
2252
2253           p0 = vlib_get_buffer (vm, pi0);
2254
2255           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2256
2257           /* We should never rewrite a pkt using the MISS adjacency */
2258           ASSERT(adj_index0);
2259
2260           adj0 = ip_get_adjacency (lm, adj_index0);
2261
2262           ip0 = vlib_buffer_get_current (p0);
2263
2264           error0 = IP6_ERROR_NONE;
2265
2266           /* Check hop limit */
2267           if (! rewrite_for_locally_received_packets)
2268             {
2269               i32 hop_limit0 = ip0->hop_limit;
2270
2271               ASSERT (ip0->hop_limit > 0);
2272
2273               hop_limit0 -= 1;
2274
2275               ip0->hop_limit = hop_limit0;
2276
2277               error0 = hop_limit0 <= 0 ? IP6_ERROR_TIME_EXPIRED : error0;
2278             }
2279
2280           if (rewrite_for_locally_received_packets)
2281             {
2282               if (PREDICT_FALSE(adj0->lookup_next_index
2283                                 == IP_LOOKUP_NEXT_LOCAL))
2284                 error0 = IP6_ERROR_SPOOFED_LOCAL_PACKETS;
2285             }
2286
2287           /* Guess we are only writing on simple Ethernet header. */
2288           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2289
2290           /* Update packet buffer attributes/set output interface. */
2291           rw_len0 = adj0[0].rewrite_header.data_bytes;
2292
2293           vlib_increment_combined_counter (&lm->adjacency_counters,
2294                                            cpu_index,
2295                                            adj_index0,
2296                                            /* packet increment */ 0,
2297                                            /* byte increment */ rw_len0);
2298
2299           /* Check MTU of outgoing interface. */
2300           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2301                     ? IP6_ERROR_MTU_EXCEEDED
2302                     : error0);
2303
2304           p0->current_data -= rw_len0;
2305           p0->current_length += rw_len0;
2306           vnet_buffer (p0)->sw_if_index[VLIB_TX] = adj0[0].rewrite_header.sw_if_index;
2307
2308           next0 = (error0 == IP6_ERROR_NONE) ?
2309             adj0[0].rewrite_header.next_index : IP6_REWRITE_NEXT_DROP;
2310
2311           p0->error = error_node->errors[error0];
2312
2313           from += 1;
2314           n_left_from -= 1;
2315           to_next += 1;
2316           n_left_to_next -= 1;
2317
2318           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2319                                            to_next, n_left_to_next,
2320                                            pi0, next0);
2321         }
2322
2323       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2324     }
2325
2326   /* Need to do trace after rewrites to pick up new packet data. */
2327   if (node->flags & VLIB_NODE_FLAG_TRACE)
2328     ip6_forward_next_trace (vm, node, frame, adj_rx_tx);
2329
2330   return frame->n_vectors;
2331 }
2332
2333 static uword
2334 ip6_rewrite_transit (vlib_main_t * vm,
2335                      vlib_node_runtime_t * node,
2336                      vlib_frame_t * frame)
2337 {
2338   return ip6_rewrite_inline (vm, node, frame,
2339                              /* rewrite_for_locally_received_packets */ 0);
2340 }
2341
2342 static uword
2343 ip6_rewrite_local (vlib_main_t * vm,
2344                    vlib_node_runtime_t * node,
2345                    vlib_frame_t * frame)
2346 {
2347   return ip6_rewrite_inline (vm, node, frame,
2348                              /* rewrite_for_locally_received_packets */ 1);
2349 }
2350
2351 VLIB_REGISTER_NODE (ip6_rewrite_node) = {
2352   .function = ip6_rewrite_transit,
2353   .name = "ip6-rewrite",
2354   .vector_size = sizeof (u32),
2355
2356   .format_trace = format_ip6_forward_next_trace,
2357
2358   .n_next_nodes = 1,
2359   .next_nodes = {
2360     [IP6_REWRITE_NEXT_DROP] = "error-drop",
2361   },
2362 };
2363
2364 VLIB_REGISTER_NODE (ip6_rewrite_local_node,static) = {
2365   .function = ip6_rewrite_local,
2366   .name = "ip6-rewrite-local",
2367   .vector_size = sizeof (u32),
2368
2369   .sibling_of = "ip6-rewrite",
2370
2371   .format_trace = format_ip6_forward_next_trace,
2372
2373   .n_next_nodes = 1,
2374   .next_nodes = {
2375     [IP6_REWRITE_NEXT_DROP] = "error-drop",
2376   },
2377 };
2378
2379 /* Global IP6 main. */
2380 ip6_main_t ip6_main;
2381
2382 static clib_error_t *
2383 ip6_lookup_init (vlib_main_t * vm)
2384 {
2385   ip6_main_t * im = &ip6_main;
2386   uword i;
2387
2388   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2389     {
2390       u32 j, i0, i1;
2391
2392       i0 = i / 32;
2393       i1 = i % 32;
2394
2395       for (j = 0; j < i0; j++)
2396         im->fib_masks[i].as_u32[j] = ~0;
2397
2398       if (i1)
2399         im->fib_masks[i].as_u32[i0] = clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2400     }
2401
2402   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2403
2404   if (im->lookup_table_nbuckets == 0)
2405     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2406
2407   im->lookup_table_nbuckets = 1<< max_log2 (im->lookup_table_nbuckets);
2408
2409   if (im->lookup_table_size == 0)
2410     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2411
2412   BV(clib_bihash_init) (&im->ip6_lookup_table, "ip6 lookup table",
2413                         im->lookup_table_nbuckets,
2414                         im->lookup_table_size);
2415
2416   /* Create FIB with index 0 and table id of 0. */
2417   find_ip6_fib_by_table_index_or_id (im, /* table id */ 0, IP6_ROUTE_FLAG_TABLE_ID);
2418
2419   {
2420     pg_node_t * pn;
2421     pn = pg_get_node (ip6_lookup_node.index);
2422     pn->unformat_edit = unformat_pg_ip6_header;
2423   }
2424
2425   {
2426     icmp6_neighbor_solicitation_header_t p;
2427
2428     memset (&p, 0, sizeof (p));
2429
2430     p.ip.ip_version_traffic_class_and_flow_label = clib_host_to_net_u32 (0x6 << 28);
2431     p.ip.payload_length = clib_host_to_net_u16 (sizeof (p)
2432                                                 - STRUCT_OFFSET_OF (icmp6_neighbor_solicitation_header_t, neighbor));
2433     p.ip.protocol = IP_PROTOCOL_ICMP6;
2434     p.ip.hop_limit = 255;
2435     ip6_set_solicited_node_multicast_address (&p.ip.dst_address, 0);
2436
2437     p.neighbor.icmp.type = ICMP6_neighbor_solicitation;
2438
2439     p.link_layer_option.header.type = ICMP6_NEIGHBOR_DISCOVERY_OPTION_source_link_layer_address;
2440     p.link_layer_option.header.n_data_u64s = sizeof (p.link_layer_option) / sizeof (u64);
2441
2442     vlib_packet_template_init (vm,
2443                                &im->discover_neighbor_packet_template,
2444                                &p, sizeof (p),
2445                                /* alloc chunk size */ 8,
2446                                "ip6 neighbor discovery");
2447   }
2448
2449   return 0;
2450 }
2451
2452 VLIB_INIT_FUNCTION (ip6_lookup_init);
2453
2454 static clib_error_t *
2455 add_del_ip6_interface_table (vlib_main_t * vm,
2456                              unformat_input_t * input,
2457                              vlib_cli_command_t * cmd)
2458 {
2459   vnet_main_t * vnm = vnet_get_main();
2460   clib_error_t * error = 0;
2461   u32 sw_if_index, table_id;
2462
2463   sw_if_index = ~0;
2464
2465   if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2466     {
2467       error = clib_error_return (0, "unknown interface `%U'",
2468                                  format_unformat_error, input);
2469       goto done;
2470     }
2471
2472   if (unformat (input, "%d", &table_id))
2473     ;
2474   else
2475     {
2476       error = clib_error_return (0, "expected table id `%U'",
2477                                  format_unformat_error, input);
2478       goto done;
2479     }
2480
2481   {
2482     ip6_main_t * im = &ip6_main;
2483     ip6_fib_t * fib =
2484       find_ip6_fib_by_table_index_or_id (im, table_id, IP6_ROUTE_FLAG_TABLE_ID);
2485
2486     if (fib)
2487       {
2488         vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
2489         im->fib_index_by_sw_if_index[sw_if_index] = fib->index;
2490     }
2491   }
2492
2493  done:
2494   return error;
2495 }
2496
2497 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) = {
2498   .path = "set interface ip6 table",
2499   .function = add_del_ip6_interface_table,
2500   .short_help = "set interface ip6 table <intfc> <table-id>"
2501 };
2502
2503 void
2504 ip6_link_local_address_from_ethernet_mac_address (ip6_address_t *ip,
2505                                                   u8 *mac)
2506 {
2507   ip->as_u64[0] = clib_host_to_net_u64 (0xFE80000000000000ULL);
2508   /* Invert the "u" bit */
2509   ip->as_u8 [8] = mac[0] ^ (1<<1);
2510   ip->as_u8 [9] = mac[1];
2511   ip->as_u8 [10] = mac[2];
2512   ip->as_u8 [11] = 0xFF;
2513   ip->as_u8 [12] = 0xFE;
2514   ip->as_u8 [13] = mac[3];
2515   ip->as_u8 [14] = mac[4];
2516   ip->as_u8 [15] = mac[5];
2517 }
2518
2519 void
2520 ip6_ethernet_mac_address_from_link_local_address (u8 *mac,
2521                                                   ip6_address_t *ip)
2522 {
2523   /* Invert the previously inverted "u" bit */
2524   mac[0] = ip->as_u8 [8] ^ (1<<1);
2525   mac[1] = ip->as_u8 [9];
2526   mac[2] = ip->as_u8 [10];
2527   mac[3] = ip->as_u8 [13];
2528   mac[4] = ip->as_u8 [14];
2529   mac[5] = ip->as_u8 [15];
2530 }
2531
2532 static clib_error_t *
2533 test_ip6_link_command_fn (vlib_main_t * vm,
2534                           unformat_input_t * input,
2535                           vlib_cli_command_t * cmd)
2536 {
2537   u8 mac[6];
2538   ip6_address_t _a, *a = &_a;
2539
2540   if (unformat (input, "%U", unformat_ethernet_address, mac))
2541     {
2542       ip6_link_local_address_from_ethernet_mac_address (a, mac);
2543       vlib_cli_output (vm, "Link local address: %U",
2544                        format_ip6_address, a);
2545       ip6_ethernet_mac_address_from_link_local_address (mac, a);
2546       vlib_cli_output (vm, "Original MAC address: %U",
2547                        format_ethernet_address, mac);
2548     }
2549
2550   return 0;
2551 }
2552
2553 VLIB_CLI_COMMAND (test_link_command, static) = {
2554   .path = "test ip6 link",
2555   .function = test_ip6_link_command_fn,
2556   .short_help = "test ip6 link <mac-address>",
2557 };
2558
2559 int vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
2560 {
2561   ip6_main_t * im6 = &ip6_main;
2562   ip6_fib_t * fib;
2563   uword * p = hash_get (im6->fib_index_by_table_id, table_id);
2564
2565   if (p == 0)
2566     return -1;
2567
2568   fib = vec_elt_at_index (im6->fibs, p[0]);
2569
2570   fib->flow_hash_config = flow_hash_config;
2571   return 1;
2572 }
2573
2574 static clib_error_t *
2575 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
2576                               unformat_input_t * input,
2577                               vlib_cli_command_t * cmd)
2578 {
2579   int matched = 0;
2580   u32 table_id = 0;
2581   u32 flow_hash_config = 0;
2582   int rv;
2583
2584   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
2585     if (unformat (input, "table %d", &table_id))
2586       matched = 1;
2587 #define _(a,v) \
2588     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2589     foreach_flow_hash_bit
2590 #undef _
2591     else break;
2592   }
2593
2594   if (matched == 0)
2595     return clib_error_return (0, "unknown input `%U'",
2596                               format_unformat_error, input);
2597
2598   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
2599   switch (rv)
2600     {
2601     case 1:
2602       break;
2603
2604     case -1:
2605       return clib_error_return (0, "no such FIB table %d", table_id);
2606
2607     default:
2608       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2609       break;
2610     }
2611
2612   return 0;
2613 }
2614
2615 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) = {
2616     .path = "set ip6 flow-hash",
2617     .short_help =
2618     "set ip table flow-hash table <fib-id> src dst sport dport proto reverse",
2619     .function = set_ip6_flow_hash_command_fn,
2620 };
2621
2622 static clib_error_t *
2623 show_ip6_local_command_fn (vlib_main_t * vm,
2624                            unformat_input_t * input,
2625                            vlib_cli_command_t * cmd)
2626 {
2627   ip6_main_t * im = &ip6_main;
2628   ip_lookup_main_t * lm = &im->lookup_main;
2629   int i;
2630
2631   vlib_cli_output (vm, "Protocols handled by ip6_local");
2632   for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
2633     {
2634       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
2635         vlib_cli_output (vm, "%d", i);
2636     }
2637   return 0;
2638 }
2639
2640
2641
2642 VLIB_CLI_COMMAND (show_ip_local, static) = {
2643   .path = "show ip6 local",
2644   .function = show_ip6_local_command_fn,
2645   .short_help = "Show ip6 local protocol table",
2646 };
2647
2648 int vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
2649                                  u32 table_index)
2650 {
2651   vnet_main_t * vnm = vnet_get_main();
2652   vnet_interface_main_t * im = &vnm->interface_main;
2653   ip6_main_t * ipm = &ip6_main;
2654   ip_lookup_main_t * lm = &ipm->lookup_main;
2655   vnet_classify_main_t * cm = &vnet_classify_main;
2656
2657   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
2658     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
2659
2660   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
2661     return VNET_API_ERROR_NO_SUCH_ENTRY;
2662
2663   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
2664   lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
2665
2666   return 0;
2667 }
2668
2669 static clib_error_t *
2670 set_ip6_classify_command_fn (vlib_main_t * vm,
2671                              unformat_input_t * input,
2672                              vlib_cli_command_t * cmd)
2673 {
2674   u32 table_index = ~0;
2675   int table_index_set = 0;
2676   u32 sw_if_index = ~0;
2677   int rv;
2678
2679   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
2680     if (unformat (input, "table-index %d", &table_index))
2681       table_index_set = 1;
2682     else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
2683                        vnet_get_main(), &sw_if_index))
2684         ;
2685     else
2686         break;
2687   }
2688
2689   if (table_index_set == 0)
2690       return clib_error_return (0, "classify table-index must be specified");
2691
2692   if (sw_if_index == ~0)
2693     return clib_error_return (0, "interface / subif must be specified");
2694
2695   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
2696
2697   switch (rv)
2698     {
2699     case 0:
2700       break;
2701
2702     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
2703       return clib_error_return (0, "No such interface");
2704
2705     case VNET_API_ERROR_NO_SUCH_ENTRY:
2706       return clib_error_return (0, "No such classifier table");
2707     }
2708   return 0;
2709 }
2710
2711 VLIB_CLI_COMMAND (set_ip6_classify_command, static) = {
2712     .path = "set ip6 classify",
2713     .short_help =
2714     "set ip6 classify intfc <int> table-index <index>",
2715     .function = set_ip6_classify_command_fn,
2716 };
2717
2718 static clib_error_t *
2719 ip6_config (vlib_main_t * vm, unformat_input_t * input)
2720 {
2721   ip6_main_t * im = &ip6_main;
2722   uword heapsize = 0;
2723   u32 tmp;
2724   u32 nbuckets = 0;
2725
2726   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
2727     if (unformat (input, "hash-buckets %d", &tmp))
2728       nbuckets = tmp;
2729     else if (unformat (input, "heap-size %dm", &tmp))
2730       heapsize = ((u64)tmp) << 20;
2731     else if (unformat (input, "heap-size %dM", &tmp))
2732       heapsize = ((u64)tmp) << 20;
2733     else if (unformat (input, "heap-size %dg", &tmp))
2734       heapsize = ((u64)tmp) << 30;
2735     else if (unformat (input, "heap-size %dG", &tmp))
2736       heapsize = ((u64)tmp) << 30;
2737     else
2738       return clib_error_return (0, "unknown input '%U'",
2739                                 format_unformat_error, input);
2740   }
2741
2742   im->lookup_table_nbuckets = nbuckets;
2743   im->lookup_table_size = heapsize;
2744
2745   return 0;
2746 }
2747
2748 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");
2749